diff --git a/Dockerfile b/Dockerfile index 490749d..4bc76b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM node:10-stretch ENV NODE_ENV="production" +ENV CHOKIDAR_USEPOLLING=1 +ENV CHOKIDAR_INTERVAL=500 VOLUME /data WORKDIR /data EXPOSE 80 diff --git a/Dockerfile_light b/Dockerfile_light index 3312543..c5df16f 100644 --- a/Dockerfile_light +++ b/Dockerfile_light @@ -1,6 +1,8 @@ FROM node:10-stretch ENV NODE_ENV="production" +ENV CHOKIDAR_USEPOLLING=1 +ENV CHOKIDAR_INTERVAL=500 EXPOSE 80 VOLUME /data WORKDIR /data diff --git a/docs/config.rst b/docs/config.rst index b7034af..3bc1a5a 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -27,6 +27,7 @@ Example:: "maxSize": 2048, "pbfAlias": "pbf", "serveAllFonts": false, + "serveAllStyles": false, "serveStaticMaps": true, "tileMargin": 0 }, @@ -99,9 +100,9 @@ Default is ``2048``. ``tileMargin`` -------------- -Additional image side length added during tile rendering that is cropped from the delivered tile. This is useful for resolving the issue with cropped labels, +Additional image side length added during tile rendering that is cropped from the delivered tile. This is useful for resolving the issue with cropped labels, but it does come with a performance degradation, because additional, adjacent vector tiles need to be loaded to genenrate a single tile. -Default is ``0`` to disable this processing. +Default is ``0`` to disable this processing. ``minRendererPoolSizes`` ------------------------ @@ -124,6 +125,13 @@ If you have plenty of memory, try setting these equal to or slightly above your If you need to conserve memory, try lower values for scale factors that are less common. Default is ``[16, 8, 4]``. +``serveAllStyles`` +------------------------ + +If this option is enabled, all the styles from the ``paths.styles`` will be served. (No recursion, only ``.json`` files are used.) +The process will also watch for changes in this directory and remove/add more styles dynamically. +It is recommended to also use the ``serveAllFonts`` option when using this option. + ``watermark`` ----------- diff --git a/package.json b/package.json index dcc7dba..0843589 100644 --- a/package.json +++ b/package.json @@ -18,15 +18,18 @@ }, "dependencies": { "@mapbox/mapbox-gl-native": "5.0.2", + "@mapbox/mapbox-gl-style-spec": "13.9.1", "@mapbox/mbtiles": "0.11.0", "@mapbox/sphericalmercator": "1.1.0", "@mapbox/vector-tile": "1.3.1", "advanced-pool": "0.3.3", "canvas": "2.6.1", + "chokidar": "3.3.1", "clone": "2.1.2", "color": "3.1.2", "commander": "4.0.1", "cors": "2.8.5", + "esm": "3.2.25", "express": "4.17.1", "glyph-pbf-composite": "0.0.2", "handlebars": "4.5.3", diff --git a/src/main.js b/src/main.js index 3dacd97..98c8a86 100644 --- a/src/main.js +++ b/src/main.js @@ -2,6 +2,8 @@ 'use strict'; +require = require('esm')(module); + const fs = require('fs'); const path = require('path'); const request = require('request'); diff --git a/src/serve_data.js b/src/serve_data.js index f083eee..37649bf 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -12,144 +12,160 @@ const VectorTile = require('@mapbox/vector-tile').VectorTile; const utils = require('./utils'); -module.exports = (options, repo, params, id, styles, publicUrl) => { - const app = express().disable('x-powered-by'); +module.exports = { + init: (options, repo) => { + const app = express().disable('x-powered-by'); - const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles); - let tileJSON = { - 'tiles': params.domains || options.domains - }; - - repo[id] = tileJSON; - - const mbtilesFileStats = fs.statSync(mbtilesFile); - if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) { - throw Error(`Not valid MBTiles file: ${mbtilesFile}`); - } - let source; - const sourceInfoPromise = new Promise((resolve, reject) => { - source = new MBTiles(mbtilesFile, err => { - if (err) { - reject(err); - return; + app.get('/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)', (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); } - source.getInfo((err, info) => { + let tileJSONFormat = item.tileJSON.format; + const z = req.params.z | 0; + const x = req.params.x | 0; + const y = req.params.y | 0; + let format = req.params.format; + if (format === options.pbfAlias) { + format = 'pbf'; + } + if (format !== tileJSONFormat && + !(format === 'geojson' && tileJSONFormat === 'pbf')) { + return res.status(404).send('Invalid format'); + } + if (z < item.tileJSON.minzoom || 0 || x < 0 || y < 0 || + z > item.tileJSON.maxzoom || + x >= Math.pow(2, z) || y >= Math.pow(2, z)) { + return res.status(404).send('Out of bounds'); + } + item.source.getTile(z, x, y, (err, data, headers) => { + let isGzipped; if (err) { - reject(err); - return; - } - tileJSON['name'] = id; - tileJSON['format'] = 'pbf'; - - Object.assign(tileJSON, info); - - tileJSON['tilejson'] = '2.0.0'; - delete tileJSON['filesize']; - delete tileJSON['mtime']; - delete tileJSON['scheme']; - - Object.assign(tileJSON, params.tilejson || {}); - utils.fixTileJSONCenter(tileJSON); - - if (options.dataDecoratorFunc) { - tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON); - } - resolve(); - }); - }); - }); - - const tilePattern = `/${id}/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)`; - - app.get(tilePattern, (req, res, next) => { - const z = req.params.z | 0; - const x = req.params.x | 0; - const y = req.params.y | 0; - let format = req.params.format; - if (format === options.pbfAlias) { - format = 'pbf'; - } - if (format !== tileJSON.format && - !(format === 'geojson' && tileJSON.format === 'pbf')) { - return res.status(404).send('Invalid format'); - } - if (z < tileJSON.minzoom || 0 || x < 0 || y < 0 || - z > tileJSON.maxzoom || - x >= Math.pow(2, z) || y >= Math.pow(2, z)) { - return res.status(404).send('Out of bounds'); - } - source.getTile(z, x, y, (err, data, headers) => { - let isGzipped; - if (err) { - if (/does not exist/.test(err.message)) { - return res.status(204).send(); + if (/does not exist/.test(err.message)) { + return res.status(204).send(); + } else { + return res.status(500).send(err.message); + } } else { - return res.status(500).send(err.message); - } - } else { - if (data == null) { - return res.status(404).send('Not found'); - } else { - if (tileJSON['format'] === 'pbf') { - isGzipped = data.slice(0, 2).indexOf( - Buffer.from([0x1f, 0x8b])) === 0; - if (options.dataDecoratorFunc) { + if (data == null) { + return res.status(404).send('Not found'); + } else { + if (tileJSONFormat === 'pbf') { + isGzipped = data.slice(0, 2).indexOf( + Buffer.from([0x1f, 0x8b])) === 0; + if (options.dataDecoratorFunc) { + if (isGzipped) { + data = zlib.unzipSync(data); + isGzipped = false; + } + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + } + } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + if (isGzipped) { data = zlib.unzipSync(data); isGzipped = false; } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - if (isGzipped) { - data = zlib.unzipSync(data); - isGzipped = false; - } - - const tile = new VectorTile(new Pbf(data)); - const geojson = { - "type": "FeatureCollection", - "features": [] - }; - for (let layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); + const tile = new VectorTile(new Pbf(data)); + const geojson = { + "type": "FeatureCollection", + "features": [] + }; + for (let layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); + } } + data = JSON.stringify(geojson); } - data = JSON.stringify(geojson); - } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); - if (!isGzipped) { - data = zlib.gzipSync(data); - isGzipped = true; - } + if (!isGzipped) { + data = zlib.gzipSync(data); + isGzipped = true; + } - return res.status(200).send(data); + return res.status(200).send(data); + } } + }); + }); + + app.get('/:id.json', (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + const info = clone(item.tileJSON); + info.tiles = utils.getTileUrls(req, info.tiles, + `data/${req.params.id}`, info.format, item.publicUrl, { + 'pbf': options.pbfAlias + }); + return res.send(info); + }); + + return app; + }, + add: (options, repo, params, id, publicUrl) => { + const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles); + let tileJSON = { + 'tiles': params.domains || options.domains + }; + + const mbtilesFileStats = fs.statSync(mbtilesFile); + if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) { + throw Error(`Not valid MBTiles file: ${mbtilesFile}`); + } + let source; + const sourceInfoPromise = new Promise((resolve, reject) => { + source = new MBTiles(mbtilesFile, err => { + if (err) { + reject(err); + return; + } + source.getInfo((err, info) => { + if (err) { + reject(err); + return; + } + tileJSON['name'] = id; + tileJSON['format'] = 'pbf'; + + Object.assign(tileJSON, info); + + tileJSON['tilejson'] = '2.0.0'; + delete tileJSON['filesize']; + delete tileJSON['mtime']; + delete tileJSON['scheme']; + + Object.assign(tileJSON, params.tilejson || {}); + utils.fixTileJSONCenter(tileJSON); + + if (options.dataDecoratorFunc) { + tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON); + } + resolve(); + }); + }); + }); + + return sourceInfoPromise.then(() => { + repo[id] = { + tileJSON, + publicUrl, + source } }); - }); - - app.get(`/${id}.json`, (req, res, next) => { - const info = clone(tileJSON); - info.tiles = utils.getTileUrls(req, info.tiles, - `data/${id}`, info.format, publicUrl, { - 'pbf': options.pbfAlias - }); - return res.send(info); - }); - - return sourceInfoPromise.then(() => app); + } }; diff --git a/src/serve_rendered.js b/src/serve_rendered.js index bbb76ac..a0536b4 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -25,6 +25,7 @@ const request = require('request'); const utils = require('./utils'); const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+\.?\\d+)'; +const httpTester = /^(http(s)?:)?\/\//; const getScale = scale => (scale || '@1x').slice(1, 2) | 0; @@ -60,7 +61,7 @@ const cachedEmptyResponses = { */ function createEmptyResponse(format, color, callback) { if (!format || format === 'pbf') { - callback(null, {data: cachedEmptyResponses['']}); + callback(null, { data: cachedEmptyResponses[''] }); return; } @@ -74,7 +75,7 @@ function createEmptyResponse(format, color, callback) { const cacheKey = `${format},${color}`; const data = cachedEmptyResponses[cacheKey]; if (data) { - callback(null, {data: data}); + callback(null, { data: data }); return; } @@ -92,702 +93,740 @@ function createEmptyResponse(format, color, callback) { if (!err) { cachedEmptyResponses[cacheKey] = buffer; } - callback(null, {data: buffer}); + callback(null, { data: buffer }); }); } -module.exports = (options, repo, params, id, publicUrl, dataResolver) => { - const app = express().disable('x-powered-by'); - - const maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); - let scalePattern = ''; - for (let i = 2; i <= maxScaleFactor; i++) { - scalePattern += i.toFixed(); - } - scalePattern = `@[${scalePattern}]x`; - - const lastModified = new Date().toUTCString(); - - const watermark = params.watermark || options.watermark; - - const styleFile = params.style; - const map = { - renderers: [], - sources: {} - }; - - const existingFonts = {}; - const fontListingPromise = new Promise((resolve, reject) => { - fs.readdir(options.paths.fonts, (err, files) => { - if (err) { - reject(err); - return; +const extractPathFromQuery = (query, transformer) => { + const pathParts = (query.path || '').split('|'); + const path = []; + for (const pair of pathParts) { + const pairParts = pair.split(','); + if (pairParts.length === 2) { + let pair; + if (query.latlng === '1' || query.latlng === 'true') { + pair = [+(pairParts[1]), +(pairParts[0])]; + } else { + pair = [+(pairParts[0]), +(pairParts[1])]; } - for (const file of files) { - fs.stat(path.join(options.paths.fonts, file), (err, stats) => { - if (err) { - reject(err); - return; - } - if (stats.isDirectory()) { - existingFonts[path.basename(file)] = true; - } - }); - } - resolve(); - }); - }); - - let styleJSON; - const createPool = (ratio, min, max) => { - const createRenderer = (ratio, createCallback) => { - const renderer = new mbgl.Map({ - mode: "tile", - ratio: ratio, - request: (req, callback) => { - const protocol = req.url.split(':')[0]; - //console.log('Handling request:', req); - if (protocol === 'sprites') { - const dir = options.paths[protocol]; - const file = unescape(req.url).substring(protocol.length + 3); - fs.readFile(path.join(dir, file), (err, data) => { - callback(err, { data: data }); - }); - } else if (protocol === 'fonts') { - const parts = req.url.split('/'); - const fontstack = unescape(parts[2]); - const range = parts[3].split('.')[0]; - utils.getFontsPbf( - null, options.paths[protocol], fontstack, range, existingFonts - ).then(concated => { - callback(null, { data: concated }); - }, err => { - callback(err, { data: null }); - }); - } else if (protocol === 'mbtiles') { - const parts = req.url.split('/'); - const sourceId = parts[2]; - const source = map.sources[sourceId]; - const sourceInfo = styleJSON.sources[sourceId]; - const z = parts[3] | 0, - x = parts[4] | 0, - y = parts[5].split('.')[0] | 0, - format = parts[5].split('.')[1]; - source.getTile(z, x, y, (err, data, headers) => { - if (err) { - if (options.verbose) console.log('MBTiles error, serving empty', err); - createEmptyResponse(sourceInfo.format, sourceInfo.color, callback); - return; - } - - const response = {}; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - try { - response.data = zlib.unzipSync(data); - } catch (err) { - console.log("Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf", id, z, x, y); - } - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, 'data', response.data, z, x, y); - } - } else { - response.data = data; - } - - callback(null, response); - }); - } else if (protocol === 'http' || protocol === 'https') { - request({ - url: req.url, - encoding: null, - gzip: true - }, (err, res, body) => { - const parts = url.parse(req.url); - const extension = path.extname(parts.pathname).toLowerCase(); - const format = extensionToFormat[extension] || ''; - if (err || res.statusCode < 200 || res.statusCode >= 300) { - // console.log('HTTP error', err || res.statusCode); - createEmptyResponse(format, '', callback); - return; - } - - const response = {}; - if (res.headers.modified) { - response.modified = new Date(res.headers.modified); - } - if (res.headers.expires) { - response.expires = new Date(res.headers.expires); - } - if (res.headers.etag) { - response.etag = res.headers.etag; - } - - response.data = body; - callback(null, response); - }); - } - } - }); - renderer.load(styleJSON); - createCallback(null, renderer); - }; - return new advancedPool.Pool({ - min: min, - max: max, - create: createRenderer.bind(null, ratio), - destroy: renderer => { - renderer.release(); - } - }); - }; - - const styleJSONPath = path.resolve(options.paths.styles, styleFile); - styleJSON = clone(require(styleJSONPath)); - - const httpTester = /^(http(s)?:)?\/\//; - if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) { - styleJSON.sprite = 'sprites://' + - styleJSON.sprite - .replace('{style}', path.basename(styleFile, '.json')) - .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleJSONPath))); - } - if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) { - styleJSON.glyphs = `fonts://${styleJSON.glyphs}`; - } - - for (const layer of (styleJSON.layers || [])) { - if (layer && layer.paint) { - // Remove (flatten) 3D buildings - if (layer.paint['fill-extrusion-height']) { - layer.paint['fill-extrusion-height'] = 0; - } - if (layer.paint['fill-extrusion-base']) { - layer.paint['fill-extrusion-base'] = 0; + if (transformer) { + pair = transformer(pair); } + path.push(pair); } } + return path; +}; - const tileJSON = { - 'tilejson': '2.0.0', - 'name': styleJSON.name, - 'attribution': '', - 'minzoom': 0, - 'maxzoom': 20, - 'bounds': [-180, -85.0511, 180, 85.0511], - 'format': 'png', - 'type': 'baselayer' +const renderOverlay = (z, x, y, bearing, pitch, w, h, scale, + path, query) => { + if (!path || path.length < 2) { + return null; + } + const precisePx = (ll, zoom) => { + const px = mercator.px(ll, 20); + const scale = Math.pow(2, zoom - 20); + return [px[0] * scale, px[1] * scale]; }; - const attributionOverride = params.tilejson && params.tilejson.attribution; - Object.assign(tileJSON, params.tilejson || {}); - tileJSON.tiles = params.domains || options.domains; - utils.fixTileJSONCenter(tileJSON); - let dataProjWGStoInternalWGS = null; + const center = precisePx([x, y], z); - const queue = []; - for (const name of Object.keys(styleJSON.sources)) { - let source = styleJSON.sources[name]; - const url = source.url; + const mapHeight = 512 * (1 << z); + const maxEdge = center[1] + h / 2; + const minEdge = center[1] - h / 2; + if (maxEdge > mapHeight) { + center[1] -= (maxEdge - mapHeight); + } else if (minEdge < 0) { + center[1] -= minEdge; + } - if (url && url.lastIndexOf('mbtiles:', 0) === 0) { - // found mbtiles source, replace with info from local file - delete source.url; + const canvas = createCanvas(scale * w, scale * h); + const ctx = canvas.getContext('2d'); + ctx.scale(scale, scale); + if (bearing) { + ctx.translate(w / 2, h / 2); + ctx.rotate(-bearing / 180 * Math.PI); + ctx.translate(-center[0], -center[1]); + } else { + // optimized path + ctx.translate(-center[0] + w / 2, -center[1] + h / 2); + } + const lineWidth = query.width !== undefined ? + parseFloat(query.width) : 1; + ctx.lineWidth = lineWidth; + ctx.strokeStyle = query.stroke || 'rgba(0,64,255,0.7)'; + ctx.fillStyle = query.fill || 'rgba(255,255,255,0.4)'; + ctx.beginPath(); + for (const pair of path) { + const px = precisePx(pair, z); + ctx.lineTo(px[0], px[1]); + } + if (path[0][0] === path[path.length - 1][0] && + path[0][1] === path[path.length - 1][1]) { + ctx.closePath(); + } + ctx.fill(); + if (lineWidth > 0) { + ctx.stroke(); + } - let mbtilesFile = url.substring('mbtiles://'.length); - const fromData = mbtilesFile[0] === '{' && - mbtilesFile[mbtilesFile.length - 1] === '}'; + return canvas.toBuffer(); +}; - if (fromData) { - mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); - const mapsTo = (params.mapping || {})[mbtilesFile]; - if (mapsTo) { - mbtilesFile = mapsTo; +const calcZForBBox = (bbox, w, h, query) => { + let z = 25; + + const padding = query.padding !== undefined ? + parseFloat(query.padding) : 0.1; + + const minCorner = mercator.px([bbox[0], bbox[3]], z), + maxCorner = mercator.px([bbox[2], bbox[1]], z); + const w_ = w / (1 + 2 * padding); + const h_ = h / (1 + 2 * padding); + + z -= Math.max( + Math.log((maxCorner[0] - minCorner[0]) / w_), + Math.log((maxCorner[1] - minCorner[1]) / h_) + ) / Math.LN2; + + z = Math.max(Math.log(Math.max(w, h) / 256) / Math.LN2, Math.min(25, z)); + + return z; +}; + +const existingFonts = {}; +let maxScaleFactor = 2; + +module.exports = { + init: (options, repo) => { + const fontListingPromise = new Promise((resolve, reject) => { + fs.readdir(options.paths.fonts, (err, files) => { + if (err) { + reject(err); + return; } - mbtilesFile = dataResolver(mbtilesFile); - if (!mbtilesFile) { - console.error(`ERROR: data "${mbtilesFile}" not found!`); - process.exit(1); - } - } - - queue.push(new Promise((resolve, reject) => { - mbtilesFile = path.resolve(options.paths.mbtiles, mbtilesFile); - const mbtilesFileStats = fs.statSync(mbtilesFile); - if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) { - throw Error(`Not valid MBTiles file: ${mbtilesFile}`); - } - map.sources[name] = new MBTiles(mbtilesFile, err => { - map.sources[name].getInfo((err, info) => { + for (const file of files) { + fs.stat(path.join(options.paths.fonts, file), (err, stats) => { if (err) { - console.error(err); + reject(err); return; } - - if (!dataProjWGStoInternalWGS && info.proj4) { - // how to do this for multiple sources with different proj4 defs? - const to3857 = proj4('EPSG:3857'); - const toDataProj = proj4(info.proj4); - dataProjWGStoInternalWGS = xy => to3857.inverse(toDataProj.forward(xy)); + if (stats.isDirectory()) { + existingFonts[path.basename(file)] = true; } - - const type = source.type; - Object.assign(source, info); - source.type = type; - source.tiles = [ - // meta url which will be detected when requested - `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}` - ]; - delete source.scheme; - - if (options.dataDecoratorFunc) { - source = options.dataDecoratorFunc(name, 'tilejson', source); - } - - if (!attributionOverride && - source.attribution && source.attribution.length > 0) { - if (tileJSON.attribution.length > 0) { - tileJSON.attribution += '; '; - } - tileJSON.attribution += source.attribution; - } - resolve(); - }); - }); - })); - } - } - - const renderersReadyPromise = Promise.all(queue).then(() => { - // standard and @2x tiles are much more usual -> default to larger pools - const minPoolSizes = options.minRendererPoolSizes || [8, 4, 2]; - const maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4]; - for (let s = 1; s <= maxScaleFactor; s++) { - const i = Math.min(minPoolSizes.length - 1, s - 1); - const j = Math.min(maxPoolSizes.length - 1, s - 1); - const minPoolSize = minPoolSizes[i]; - const maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]); - map.renderers[s] = createPool(s, minPoolSize, maxPoolSize); - } - }); - - repo[id] = tileJSON; - - const tilePattern = `/${id}/:z(\\d+)/:x(\\d+)/:y(\\d+):scale(${scalePattern})?.:format([\\w]+)`; - - const respondImage = (z, lon, lat, bearing, pitch, - width, height, scale, format, res, next, - opt_overlay) => { - if (Math.abs(lon) > 180 || Math.abs(lat) > 85.06 || - lon !== lon || lat !== lat) { - return res.status(400).send('Invalid center'); - } - if (Math.min(width, height) <= 0 || - Math.max(width, height) * scale > (options.maxSize || 2048) || - width !== width || height !== height) { - return res.status(400).send('Invalid size'); - } - if (format === 'png' || format === 'webp') { - } else if (format === 'jpg' || format === 'jpeg') { - format = 'jpeg'; - } else { - return res.status(400).send('Invalid format'); - } - - const pool = map.renderers[scale]; - pool.acquire((err, renderer) => { - const mbglZ = Math.max(0, z - 1); - const params = { - zoom: mbglZ, - center: [lon, lat], - bearing: bearing, - pitch: pitch, - width: width, - height: height - }; - if (z === 0) { - params.width *= 2; - params.height *= 2; - } - - const tileMargin = Math.max(options.tileMargin || 0, 0); - if (z > 2 && tileMargin > 0) { - params.width += tileMargin * 2; - params.height += tileMargin * 2; - } - - renderer.render(params, (err, data) => { - pool.release(renderer); - if (err) { - console.error(err); - return res.status(500).send(err); - } - - // Fix semi-transparent outlines on raw, premultiplied input - // https://github.com/klokantech/tileserver-gl/issues/350#issuecomment-477857040 - for (var i = 0; i < data.length; i += 4) { - var alpha = data[i + 3]; - var norm = alpha / 255; - if (alpha === 0) { - data[i] = 0; - data[i + 1] = 0; - data[i + 2] = 0; - } else { - data[i] = data[i] / norm; - data[i + 1] = data[i + 1] / norm; - data[i + 2] = data[i + 2] / norm; - } - } - - const image = sharp(data, { - raw: { - width: params.width * scale, - height: params.height * scale, - channels: 4 - } - }); - - if (z > 2 && tileMargin > 0) { - image.extract({ - left: tileMargin * scale, - top: tileMargin * scale, - width: width * scale, - height: height * scale }); } - - if (z === 0) { - // HACK: when serving zoom 0, resize the 0 tile from 512 to 256 - image.resize(width * scale, height * scale); - } - - if (opt_overlay) { - image.composite([{ input: opt_overlay }]); - } - if (watermark) { - const canvas = createCanvas(scale * width, scale * height); - const ctx = canvas.getContext('2d'); - ctx.scale(scale, scale); - ctx.font = '10px sans-serif'; - ctx.strokeWidth = '1px'; - ctx.strokeStyle = 'rgba(255,255,255,.4)'; - ctx.strokeText(watermark, 5, height - 5); - ctx.fillStyle = 'rgba(0,0,0,.4)'; - ctx.fillText(watermark, 5, height - 5); - - image.composite([{ input: canvas.toBuffer() }]); - } - - const formatQuality = (params.formatQuality || {})[format] || - (options.formatQuality || {})[format]; - - if (format === 'png') { - image.png({ adaptiveFiltering: false }); - } else if (format === 'jpeg') { - image.jpeg({ quality: formatQuality || 80 }); - } else if (format === 'webp') { - image.webp({ quality: formatQuality || 90 }); - } - image.toBuffer((err, buffer, info) => { - if (!buffer) { - return res.status(404).send('Not found'); - } - - res.set({ - 'Last-Modified': lastModified, - 'Content-Type': `image/${format}` - }); - return res.status(200).send(buffer); - }); + resolve(); }); }); - }; - app.get(tilePattern, (req, res, next) => { - const modifiedSince = req.get('if-modified-since'), cc = req.get('cache-control'); - if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { - if (new Date(lastModified) <= new Date(modifiedSince)) { - return res.sendStatus(304); - } + maxScaleFactor = Math.min(Math.floor(options.maxScaleFactor || 3), 9); + let scalePattern = ''; + for (let i = 2; i <= maxScaleFactor; i++) { + scalePattern += i.toFixed(); } + scalePattern = `@[${scalePattern}]x`; - const z = req.params.z | 0, - x = req.params.x | 0, - y = req.params.y | 0, - scale = getScale(req.params.scale), - format = req.params.format; - if (z < 0 || x < 0 || y < 0 || + const app = express().disable('x-powered-by'); + + const respondImage = (item, z, lon, lat, bearing, pitch, + width, height, scale, format, res, next, + opt_overlay) => { + if (Math.abs(lon) > 180 || Math.abs(lat) > 85.06 || + lon !== lon || lat !== lat) { + return res.status(400).send('Invalid center'); + } + if (Math.min(width, height) <= 0 || + Math.max(width, height) * scale > (options.maxSize || 2048) || + width !== width || height !== height) { + return res.status(400).send('Invalid size'); + } + if (format === 'png' || format === 'webp') { + } else if (format === 'jpg' || format === 'jpeg') { + format = 'jpeg'; + } else { + return res.status(400).send('Invalid format'); + } + + const pool = item.map.renderers[scale]; + pool.acquire((err, renderer) => { + const mbglZ = Math.max(0, z - 1); + const params = { + zoom: mbglZ, + center: [lon, lat], + bearing: bearing, + pitch: pitch, + width: width, + height: height + }; + if (z === 0) { + params.width *= 2; + params.height *= 2; + } + + const tileMargin = Math.max(options.tileMargin || 0, 0); + if (z > 2 && tileMargin > 0) { + params.width += tileMargin * 2; + params.height += tileMargin * 2; + } + + renderer.render(params, (err, data) => { + pool.release(renderer); + if (err) { + console.error(err); + return; + } + + // Fix semi-transparent outlines on raw, premultiplied input + // https://github.com/klokantech/tileserver-gl/issues/350#issuecomment-477857040 + for (var i = 0; i < data.length; i += 4) { + var alpha = data[i + 3]; + var norm = alpha / 255; + if (alpha === 0) { + data[i] = 0; + data[i + 1] = 0; + data[i + 2] = 0; + } else { + data[i] = data[i] / norm; + data[i + 1] = data[i + 1] / norm; + data[i + 2] = data[i + 2] / norm; + } + } + + const image = sharp(data, { + raw: { + width: params.width * scale, + height: params.height * scale, + channels: 4 + } + }); + + if (z > 2 && tileMargin > 0) { + image.extract({ + left: tileMargin * scale, + top: tileMargin * scale, + width: width * scale, + height: height * scale + }); + } + + if (z === 0) { + // HACK: when serving zoom 0, resize the 0 tile from 512 to 256 + image.resize(width * scale, height * scale); + } + + if (opt_overlay) { + image.composite([{ input: opt_overlay }]); + } + if (item.watermark) { + const canvas = createCanvas(scale * width, scale * height); + const ctx = canvas.getContext('2d'); + ctx.scale(scale, scale); + ctx.font = '10px sans-serif'; + ctx.strokeWidth = '1px'; + ctx.strokeStyle = 'rgba(255,255,255,.4)'; + ctx.strokeText(item.watermark, 5, height - 5); + ctx.fillStyle = 'rgba(0,0,0,.4)'; + ctx.fillText(item.watermark, 5, height - 5); + + image.composite([{ input: canvas.toBuffer() }]); + } + + const formatQuality = (options.formatQuality || {})[format]; + + if (format === 'png') { + image.png({ adaptiveFiltering: false }); + } else if (format === 'jpeg') { + image.jpeg({ quality: formatQuality || 80 }); + } else if (format === 'webp') { + image.webp({ quality: formatQuality || 90 }); + } + image.toBuffer((err, buffer, info) => { + if (!buffer) { + return res.status(404).send('Not found'); + } + + res.set({ + 'Last-Modified': item.lastModified, + 'Content-Type': `image/${format}` + }); + return res.status(200).send(buffer); + }); + }); + }); + }; + + app.get(`/:id/:z(\\d+)/:x(\\d+)/:y(\\d+):scale(${scalePattern})?.:format([\\w]+)`, (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + + const modifiedSince = req.get('if-modified-since'), cc = req.get('cache-control'); + if (modifiedSince && (!cc || cc.indexOf('no-cache') === -1)) { + if (new Date(item.lastModified) <= new Date(modifiedSince)) { + return res.sendStatus(304); + } + } + + const z = req.params.z | 0, + x = req.params.x | 0, + y = req.params.y | 0, + scale = getScale(req.params.scale), + format = req.params.format; + if (z < 0 || x < 0 || y < 0 || z > 20 || x >= Math.pow(2, z) || y >= Math.pow(2, z)) { - return res.status(404).send('Out of bounds'); - } - const tileSize = 256; - const tileCenter = mercator.ll([ - ((x + 0.5) / (1 << z)) * (256 << z), - ((y + 0.5) / (1 << z)) * (256 << z) - ], z); - return respondImage(z, tileCenter[0], tileCenter[1], 0, 0, - tileSize, tileSize, scale, format, res, next); - }); + return res.status(404).send('Out of bounds'); + } + const tileSize = 256; + const tileCenter = mercator.ll([ + ((x + 0.5) / (1 << z)) * (256 << z), + ((y + 0.5) / (1 << z)) * (256 << z) + ], z); + return respondImage(item, z, tileCenter[0], tileCenter[1], 0, 0, + tileSize, tileSize, scale, format, res, next); + }); - const extractPathFromQuery = (query, transformer) => { - const pathParts = (query.path || '').split('|'); - const path = []; - for (const pair of pathParts) { - const pairParts = pair.split(','); - if (pairParts.length === 2) { - let pair; - if (query.latlng === '1' || query.latlng === 'true') { - pair = [+(pairParts[1]), +(pairParts[0])]; - } else { - pair = [+(pairParts[0]), +(pairParts[1])]; + if (options.serveStaticMaps !== false) { + const staticPattern = + `/:id/static/:raw(raw)?/%s/:width(\\d+)x:height(\\d+):scale(${scalePattern})?.:format([\\w]+)`; + + const centerPattern = + util.format(':x(%s),:y(%s),:z(%s)(@:bearing(%s)(,:pitch(%s))?)?', + FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN, + FLOAT_PATTERN, FLOAT_PATTERN); + + app.get(util.format(staticPattern, centerPattern), (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); } + const raw = req.params.raw; + let z = +req.params.z, + x = +req.params.x, + y = +req.params.y, + bearing = +(req.params.bearing || '0'), + pitch = +(req.params.pitch || '0'), + w = req.params.width | 0, + h = req.params.height | 0, + scale = getScale(req.params.scale), + format = req.params.format; + + if (z < 0) { + return res.status(404).send('Invalid zoom'); + } + + const transformer = raw ? + mercator.inverse.bind(mercator) : item.dataProjWGStoInternalWGS; + if (transformer) { - pair = transformer(pair); + const ll = transformer([x, y]); + x = ll[0]; + y = ll[1]; } - path.push(pair); - } - } - return path; - }; - const renderOverlay = (z, x, y, bearing, pitch, w, h, scale, - path, query) => { - if (!path || path.length < 2) { - return null; - } - const precisePx = (ll, zoom) => { - const px = mercator.px(ll, 20); - const scale = Math.pow(2, zoom - 20); - return [px[0] * scale, px[1] * scale]; - }; + const path = extractPathFromQuery(req.query, transformer); + const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale, + path, req.query); - const center = precisePx([x, y], z); + return respondImage(item, z, x, y, bearing, pitch, w, h, scale, format, + res, next, overlay); + }); - const mapHeight = 512 * (1 << z); - const maxEdge = center[1] + h / 2; - const minEdge = center[1] - h / 2; - if (maxEdge > mapHeight) { - center[1] -= (maxEdge - mapHeight); - } else if (minEdge < 0) { - center[1] -= minEdge; - } - - const canvas = createCanvas(scale * w, scale * h); - const ctx = canvas.getContext('2d'); - ctx.scale(scale, scale); - if (bearing) { - ctx.translate(w / 2, h / 2); - ctx.rotate(-bearing / 180 * Math.PI); - ctx.translate(-center[0], -center[1]); - } else { - // optimized path - ctx.translate(-center[0] + w / 2, -center[1] + h / 2); - } - const lineWidth = query.width !== undefined ? - parseFloat(query.width) : 1; - ctx.lineWidth = lineWidth; - ctx.strokeStyle = query.stroke || 'rgba(0,64,255,0.7)'; - ctx.fillStyle = query.fill || 'rgba(255,255,255,0.4)'; - ctx.beginPath(); - for (const pair of path) { - const px = precisePx(pair, z); - ctx.lineTo(px[0], px[1]); - } - if (path[0][0] === path[path.length - 1][0] && - path[0][1] === path[path.length - 1][1]) { - ctx.closePath(); - } - ctx.fill(); - if (lineWidth > 0) { - ctx.stroke(); - } - - return canvas.toBuffer(); - }; - - const calcZForBBox = (bbox, w, h, query) => { - let z = 25; - - const padding = query.padding !== undefined ? - parseFloat(query.padding) : 0.1; - - const minCorner = mercator.px([bbox[0], bbox[3]], z), - maxCorner = mercator.px([bbox[2], bbox[1]], z); - const w_ = w / (1 + 2 * padding); - const h_ = h / (1 + 2 * padding); - - z -= Math.max( - Math.log((maxCorner[0] - minCorner[0]) / w_), - Math.log((maxCorner[1] - minCorner[1]) / h_) - ) / Math.LN2; - - z = Math.max(Math.log(Math.max(w, h) / 256) / Math.LN2, Math.min(25, z)); - - return z; - }; - - if (options.serveStaticMaps !== false) { - const staticPattern = - `/${id}/static/:raw(raw)?/%s/:width(\\d+)x:height(\\d+):scale(${scalePattern})?.:format([\\w]+)`; - - const centerPattern = - util.format(':x(%s),:y(%s),:z(%s)(@:bearing(%s)(,:pitch(%s))?)?', - FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN, - FLOAT_PATTERN, FLOAT_PATTERN); - - app.get(util.format(staticPattern, centerPattern), (req, res, next) => { - const raw = req.params.raw; - let z = +req.params.z, - x = +req.params.x, - y = +req.params.y, - bearing = +(req.params.bearing || '0'), - pitch = +(req.params.pitch || '0'), - w = req.params.width | 0, - h = req.params.height | 0, - scale = getScale(req.params.scale), - format = req.params.format; - - if (z < 0) { - return res.status(404).send('Invalid zoom'); - } - - const transformer = raw ? - mercator.inverse.bind(mercator) : dataProjWGStoInternalWGS; - - if (transformer) { - const ll = transformer([x, y]); - x = ll[0]; - y = ll[1]; - } - - const path = extractPathFromQuery(req.query, transformer); - const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale, - path, req.query); - - return respondImage(z, x, y, bearing, pitch, w, h, scale, format, - res, next, overlay); - }); - - const serveBounds = (req, res, next) => { - const raw = req.params.raw; - const bbox = [+req.params.minx, +req.params.miny, + const serveBounds = (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + const raw = req.params.raw; + const bbox = [+req.params.minx, +req.params.miny, +req.params.maxx, +req.params.maxy]; - let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; + let center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]; - const transformer = raw ? - mercator.inverse.bind(mercator) : dataProjWGStoInternalWGS; + const transformer = raw ? + mercator.inverse.bind(mercator) : item.dataProjWGStoInternalWGS; - if (transformer) { - const minCorner = transformer(bbox.slice(0, 2)); - const maxCorner = transformer(bbox.slice(2)); - bbox[0] = minCorner[0]; - bbox[1] = minCorner[1]; - bbox[2] = maxCorner[0]; - bbox[3] = maxCorner[1]; - center = transformer(center); + if (transformer) { + const minCorner = transformer(bbox.slice(0, 2)); + const maxCorner = transformer(bbox.slice(2)); + bbox[0] = minCorner[0]; + bbox[1] = minCorner[1]; + bbox[2] = maxCorner[0]; + bbox[3] = maxCorner[1]; + center = transformer(center); + } + + const w = req.params.width | 0, + h = req.params.height | 0, + scale = getScale(req.params.scale), + format = req.params.format; + + const z = calcZForBBox(bbox, w, h, req.query), + x = center[0], + y = center[1], + bearing = 0, + pitch = 0; + + const path = extractPathFromQuery(req.query, transformer); + const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale, + path, req.query); + return respondImage(item, z, x, y, bearing, pitch, w, h, scale, format, + res, next, overlay); + }; + + const boundsPattern = + util.format(':minx(%s),:miny(%s),:maxx(%s),:maxy(%s)', + FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN); + + app.get(util.format(staticPattern, boundsPattern), serveBounds); + + app.get('/:id/static/', (req, res, next) => { + for (let key in req.query) { + req.query[key.toLowerCase()] = req.query[key]; + } + req.params.raw = true; + req.params.format = (req.query.format || 'image/png').split('/').pop(); + const bbox = (req.query.bbox || '').split(','); + req.params.minx = bbox[0]; + req.params.miny = bbox[1]; + req.params.maxx = bbox[2]; + req.params.maxy = bbox[3]; + req.params.width = req.query.width || '256'; + req.params.height = req.query.height || '256'; + if (req.query.scale) { + req.params.width /= req.query.scale; + req.params.height /= req.query.scale; + req.params.scale = `@${req.query.scale}`; + } + + return serveBounds(req, res, next); + }); + + const autoPattern = 'auto'; + + app.get(util.format(staticPattern, autoPattern), (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + const raw = req.params.raw; + const w = req.params.width | 0, + h = req.params.height | 0, + bearing = 0, + pitch = 0, + scale = getScale(req.params.scale), + format = req.params.format; + + const transformer = raw ? + mercator.inverse.bind(mercator) : item.dataProjWGStoInternalWGS; + + const path = extractPathFromQuery(req.query, transformer); + if (path.length < 2) { + return res.status(400).send('Invalid path'); + } + + const bbox = [Infinity, Infinity, -Infinity, -Infinity]; + for (const pair of path) { + bbox[0] = Math.min(bbox[0], pair[0]); + bbox[1] = Math.min(bbox[1], pair[1]); + bbox[2] = Math.max(bbox[2], pair[0]); + bbox[3] = Math.max(bbox[3], pair[1]); + } + + const bbox_ = mercator.convert(bbox, '900913'); + const center = mercator.inverse( + [(bbox_[0] + bbox_[2]) / 2, (bbox_[1] + bbox_[3]) / 2] + ); + + const z = calcZForBBox(bbox, w, h, req.query), + x = center[0], + y = center[1]; + + const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale, + path, req.query); + + return respondImage(item, z, x, y, bearing, pitch, w, h, scale, format, + res, next, overlay); + }); + } + + app.get('/:id.json', (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); } + const info = clone(item.tileJSON); + info.tiles = utils.getTileUrls(req, info.tiles, + `styles/${req.params.id}`, info.format, item.publicUrl); + return res.send(info); + }); - const w = req.params.width | 0, - h = req.params.height | 0, - scale = getScale(req.params.scale), - format = req.params.format; - - const z = calcZForBBox(bbox, w, h, req.query), - x = center[0], - y = center[1], - bearing = 0, - pitch = 0; - - const path = extractPathFromQuery(req.query, transformer); - const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale, - path, req.query); - return respondImage(z, x, y, bearing, pitch, w, h, scale, format, - res, next, overlay); + return Promise.all([fontListingPromise]).then(() => app); + }, + add: (options, repo, params, id, publicUrl, dataResolver) => { + const map = { + renderers: [], + sources: {} }; - const boundsPattern = - util.format(':minx(%s),:miny(%s),:maxx(%s),:maxy(%s)', - FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN, FLOAT_PATTERN); + let styleJSON; + const createPool = (ratio, min, max) => { + const createRenderer = (ratio, createCallback) => { + const renderer = new mbgl.Map({ + mode: "tile", + ratio: ratio, + request: (req, callback) => { + const protocol = req.url.split(':')[0]; + //console.log('Handling request:', req); + if (protocol === 'sprites') { + const dir = options.paths[protocol]; + const file = unescape(req.url).substring(protocol.length + 3); + fs.readFile(path.join(dir, file), (err, data) => { + callback(err, { data: data }); + }); + } else if (protocol === 'fonts') { + const parts = req.url.split('/'); + const fontstack = unescape(parts[2]); + const range = parts[3].split('.')[0]; + utils.getFontsPbf( + null, options.paths[protocol], fontstack, range, existingFonts + ).then(concated => { + callback(null, { data: concated }); + }, err => { + callback(err, { data: null }); + }); + } else if (protocol === 'mbtiles') { + const parts = req.url.split('/'); + const sourceId = parts[2]; + const source = map.sources[sourceId]; + const sourceInfo = styleJSON.sources[sourceId]; + const z = parts[3] | 0, + x = parts[4] | 0, + y = parts[5].split('.')[0] | 0, + format = parts[5].split('.')[1]; + source.getTile(z, x, y, (err, data, headers) => { + if (err) { + if (options.verbose) console.log('MBTiles error, serving empty', err); + createEmptyResponse(sourceInfo.format, sourceInfo.color, callback); + return; + } - app.get(util.format(staticPattern, boundsPattern), serveBounds); + const response = {}; + if (headers['Last-Modified']) { + response.modified = new Date(headers['Last-Modified']); + } - app.get(`/${id}/static/`, (req, res, next) => { - for (let key in req.query) { - req.query[key.toLowerCase()] = req.query[key]; - } - req.params.raw = true; - req.params.format = (req.query.format || 'image/png').split('/').pop(); - const bbox = (req.query.bbox || '').split(','); - req.params.minx = bbox[0]; - req.params.miny = bbox[1]; - req.params.maxx = bbox[2]; - req.params.maxy = bbox[3]; - req.params.width = req.query.width || '256'; - req.params.height = req.query.height || '256'; - if (req.query.scale) { - req.params.width /= req.query.scale; - req.params.height /= req.query.scale; - req.params.scale = `@${req.query.scale}`; + if (format === 'pbf') { + try { + response.data = zlib.unzipSync(data); + } catch (err) { + console.log("Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf", id, z, x, y); + } + if (options.dataDecoratorFunc) { + response.data = options.dataDecoratorFunc( + sourceId, 'data', response.data, z, x, y); + } + } else { + response.data = data; + } + + callback(null, response); + }); + } else if (protocol === 'http' || protocol === 'https') { + request({ + url: req.url, + encoding: null, + gzip: true + }, (err, res, body) => { + const parts = url.parse(req.url); + const extension = path.extname(parts.pathname).toLowerCase(); + const format = extensionToFormat[extension] || ''; + if (err || res.statusCode < 200 || res.statusCode >= 300) { + // console.log('HTTP error', err || res.statusCode); + createEmptyResponse(format, '', callback); + return; + } + + const response = {}; + if (res.headers.modified) { + response.modified = new Date(res.headers.modified); + } + if (res.headers.expires) { + response.expires = new Date(res.headers.expires); + } + if (res.headers.etag) { + response.etag = res.headers.etag; + } + + response.data = body; + callback(null, response); + }); + } + } + }); + renderer.load(styleJSON); + createCallback(null, renderer); + }; + return new advancedPool.Pool({ + min: min, + max: max, + create: createRenderer.bind(null, ratio), + destroy: renderer => { + renderer.release(); + } + }); + }; + + const styleFile = params.style; + const styleJSONPath = path.resolve(options.paths.styles, styleFile); + try { + styleJSON = JSON.parse(fs.readFileSync(styleJSONPath)); + } catch (e) { + console.log('Error parsing style file'); + return false; + } + + if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) { + styleJSON.sprite = 'sprites://' + + styleJSON.sprite + .replace('{style}', path.basename(styleFile, '.json')) + .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleJSONPath))); + } + if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) { + styleJSON.glyphs = `fonts://${styleJSON.glyphs}`; + } + + for (const layer of (styleJSON.layers || [])) { + if (layer && layer.paint) { + // Remove (flatten) 3D buildings + if (layer.paint['fill-extrusion-height']) { + layer.paint['fill-extrusion-height'] = 0; + } + if (layer.paint['fill-extrusion-base']) { + layer.paint['fill-extrusion-base'] = 0; + } } + } - return serveBounds(req, res, next); + const tileJSON = { + 'tilejson': '2.0.0', + 'name': styleJSON.name, + 'attribution': '', + 'minzoom': 0, + 'maxzoom': 20, + 'bounds': [-180, -85.0511, 180, 85.0511], + 'format': 'png', + 'type': 'baselayer' + }; + const attributionOverride = params.tilejson && params.tilejson.attribution; + Object.assign(tileJSON, params.tilejson || {}); + tileJSON.tiles = params.domains || options.domains; + utils.fixTileJSONCenter(tileJSON); + + repo[id] = { + tileJSON, + publicUrl, + map, + dataProjWGStoInternalWGS: null, + lastModified: new Date().toUTCString(), + watermark: params.watermark || options.watermark + }; + + const queue = []; + for (const name of Object.keys(styleJSON.sources)) { + let source = styleJSON.sources[name]; + const url = source.url; + + if (url && url.lastIndexOf('mbtiles:', 0) === 0) { + // found mbtiles source, replace with info from local file + delete source.url; + + let mbtilesFile = url.substring('mbtiles://'.length); + const fromData = mbtilesFile[0] === '{' && + mbtilesFile[mbtilesFile.length - 1] === '}'; + + if (fromData) { + mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); + const mapsTo = (params.mapping || {})[mbtilesFile]; + if (mapsTo) { + mbtilesFile = mapsTo; + } + mbtilesFile = dataResolver(mbtilesFile); + if (!mbtilesFile) { + console.error(`ERROR: data "${mbtilesFile}" not found!`); + process.exit(1); + } + } + + queue.push(new Promise((resolve, reject) => { + mbtilesFile = path.resolve(options.paths.mbtiles, mbtilesFile); + const mbtilesFileStats = fs.statSync(mbtilesFile); + if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) { + throw Error(`Not valid MBTiles file: ${mbtilesFile}`); + } + map.sources[name] = new MBTiles(mbtilesFile, err => { + map.sources[name].getInfo((err, info) => { + if (err) { + console.error(err); + return; + } + + if (!repo[id].dataProjWGStoInternalWGS && info.proj4) { + // how to do this for multiple sources with different proj4 defs? + const to3857 = proj4('EPSG:3857'); + const toDataProj = proj4(info.proj4); + repo[id].dataProjWGStoInternalWGS = xy => to3857.inverse(toDataProj.forward(xy)); + } + + const type = source.type; + Object.assign(source, info); + source.type = type; + source.tiles = [ + // meta url which will be detected when requested + `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}` + ]; + delete source.scheme; + + if (options.dataDecoratorFunc) { + source = options.dataDecoratorFunc(name, 'tilejson', source); + } + + if (!attributionOverride && + source.attribution && source.attribution.length > 0) { + if (tileJSON.attribution.length > 0) { + tileJSON.attribution += '; '; + } + tileJSON.attribution += source.attribution; + } + resolve(); + }); + }); + })); + } + } + + const renderersReadyPromise = Promise.all(queue).then(() => { + // standard and @2x tiles are much more usual -> default to larger pools + const minPoolSizes = options.minRendererPoolSizes || [8, 4, 2]; + const maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4]; + for (let s = 1; s <= maxScaleFactor; s++) { + const i = Math.min(minPoolSizes.length - 1, s - 1); + const j = Math.min(maxPoolSizes.length - 1, s - 1); + const minPoolSize = minPoolSizes[i]; + const maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]); + map.renderers[s] = createPool(s, minPoolSize, maxPoolSize); + } }); - const autoPattern = 'auto'; - - app.get(util.format(staticPattern, autoPattern), (req, res, next) => { - const raw = req.params.raw; - const w = req.params.width | 0, - h = req.params.height | 0, - bearing = 0, - pitch = 0, - scale = getScale(req.params.scale), - format = req.params.format; - - const transformer = raw ? - mercator.inverse.bind(mercator) : dataProjWGStoInternalWGS; - - const path = extractPathFromQuery(req.query, transformer); - if (path.length < 2) { - return res.status(400).send('Invalid path'); - } - - const bbox = [Infinity, Infinity, -Infinity, -Infinity]; - for (const pair of path) { - bbox[0] = Math.min(bbox[0], pair[0]); - bbox[1] = Math.min(bbox[1], pair[1]); - bbox[2] = Math.max(bbox[2], pair[0]); - bbox[3] = Math.max(bbox[3], pair[1]); - } - - const bbox_ = mercator.convert(bbox, '900913'); - const center = mercator.inverse( - [(bbox_[0] + bbox_[2]) / 2, (bbox_[1] + bbox_[3]) / 2] - ); - - const z = calcZForBBox(bbox, w, h, req.query), - x = center[0], - y = center[1]; - - const overlay = renderOverlay(z, x, y, bearing, pitch, w, h, scale, - path, req.query); - - return respondImage(z, x, y, bearing, pitch, w, h, scale, format, - res, next, overlay); - }); - } - - app.get(`/${id}.json`, (req, res, next) => { - const info = clone(tileJSON); - info.tiles = utils.getTileUrls(req, info.tiles, - `styles/${id}`, info.format, publicUrl); - return res.send(info); - }); - - return Promise.all([fontListingPromise, renderersReadyPromise]).then(() => app); - + return Promise.all([renderersReadyPromise]); + }, + remove: (repo, id) => { + let item = repo[id]; + if (item) { + item.map.renderers.forEach(pool => { + pool.close(); + }); + } + delete repo[id]; + }, }; diff --git a/src/serve_style.js b/src/serve_style.js index 65b9669..ca441db 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -5,115 +5,154 @@ const fs = require('fs'); const clone = require('clone'); const express = require('express'); +import {validate} from '@mapbox/mapbox-gl-style-spec'; const utils = require('./utils'); -module.exports = (options, repo, params, id, publicUrl, reportTiles, reportFont) => { - const app = express().disable('x-powered-by'); +const httpTester = /^(http(s)?:)?\/\//; - const styleFile = path.resolve(options.paths.styles, params.style); +const fixUrl = (req, url, publicUrl, opt_nokey) => { + if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) { + return url; + } + const queryParams = []; + if (!opt_nokey && req.query.key) { + queryParams.unshift(`key=${req.query.key}`); + } + let query = ''; + if (queryParams.length) { + query = `?${queryParams.join('&')}`; + } + return url.replace( + 'local://', utils.getPublicUrl(publicUrl, req)) + query; +}; - const styleJSON = clone(require(styleFile)); - for (const name of Object.keys(styleJSON.sources)) { - const source = styleJSON.sources[name]; - const url = source.url; - if (url && url.lastIndexOf('mbtiles:', 0) === 0) { - let mbtilesFile = url.substring('mbtiles://'.length); - const fromData = mbtilesFile[0] === '{' && - mbtilesFile[mbtilesFile.length - 1] === '}'; +module.exports = { + init: (options, repo) => { + const app = express().disable('x-powered-by'); - if (fromData) { - mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); - const mapsTo = (params.mapping || {})[mbtilesFile]; - if (mapsTo) { - mbtilesFile = mapsTo; + app.get('/:id/style.json', (req, res, next) => { + const item = repo[req.params.id]; + if (!item) { + return res.sendStatus(404); + } + const styleJSON_ = clone(item.styleJSON); + for (const name of Object.keys(styleJSON_.sources)) { + const source = styleJSON_.sources[name]; + source.url = fixUrl(req, source.url, item.publicUrl); + } + // mapbox-gl-js viewer cannot handle sprite urls with query + if (styleJSON_.sprite) { + styleJSON_.sprite = fixUrl(req, styleJSON_.sprite, item.publicUrl, true); + } + if (styleJSON_.glyphs) { + styleJSON_.glyphs = fixUrl(req, styleJSON_.glyphs, item.publicUrl, false); + } + return res.send(styleJSON_); + }); + + app.get('/:id/sprite:scale(@[23]x)?.:format([\\w]+)', (req, res, next) => { + const item = repo[req.params.id]; + if (!item || !item.spritePath) { + return res.sendStatus(404); + } + const scale = req.params.scale, + format = req.params.format; + const filename = `${item.spritePath + (scale || '')}.${format}`; + return fs.readFile(filename, (err, data) => { + if (err) { + console.log('Sprite load error:', filename); + return res.sendStatus(404); + } else { + if (format === 'json') res.header('Content-type', 'application/json'); + if (format === 'png') res.header('Content-type', 'image/png'); + return res.send(data); + } + }); + }); + + return app; + }, + remove: (repo, id) => { + delete repo[id]; + }, + add: (options, repo, params, id, publicUrl, reportTiles, reportFont) => { + const styleFile = path.resolve(options.paths.styles, params.style); + + let styleFileData; + try { + styleFileData = fs.readFileSync(styleFile); + } catch (e) { + console.log('Error reading style file'); + return false; + } + + let validationErrors = validate(styleFileData); + if (validationErrors.length > 0) { + console.log(`The file "${params.style}" is not valid a valid style file:`); + for (const err of validationErrors) { + console.log(`${err.line}: ${err.message}`); + } + return false; + } + let styleJSON = JSON.parse(styleFileData); + + for (const name of Object.keys(styleJSON.sources)) { + const source = styleJSON.sources[name]; + const url = source.url; + if (url && url.lastIndexOf('mbtiles:', 0) === 0) { + let mbtilesFile = url.substring('mbtiles://'.length); + const fromData = mbtilesFile[0] === '{' && + mbtilesFile[mbtilesFile.length - 1] === '}'; + + if (fromData) { + mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); + const mapsTo = (params.mapping || {})[mbtilesFile]; + if (mapsTo) { + mbtilesFile = mapsTo; + } + } + const identifier = reportTiles(mbtilesFile, fromData); + if (!identifier) { + return false; + } + source.url = `local://data/${identifier}.json`; + } + } + + for (let obj of styleJSON.layers) { + if (obj['type'] === 'symbol') { + const fonts = (obj['layout'] || {})['text-font']; + if (fonts && fonts.length) { + fonts.forEach(reportFont); + } else { + reportFont('Open Sans Regular'); + reportFont('Arial Unicode MS Regular'); } } - const identifier = reportTiles(mbtilesFile, fromData); - source.url = `local://data/${identifier}.json`; } - } - for(let obj of styleJSON.layers) { - if (obj['type'] === 'symbol') { - const fonts = (obj['layout'] || {})['text-font']; - if (fonts && fonts.length) { - fonts.forEach(reportFont); - } else { - reportFont('Open Sans Regular'); - reportFont('Arial Unicode MS Regular'); - } - } - } + let spritePath; - let spritePath; - - const httpTester = /^(http(s)?:)?\/\//; - if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) { - spritePath = path.join(options.paths.sprites, + if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) { + spritePath = path.join(options.paths.sprites, styleJSON.sprite - .replace('{style}', path.basename(styleFile, '.json')) - .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile))) - ); - styleJSON.sprite = `local://styles/${id}/sprite`; - } - if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) { - styleJSON.glyphs = 'local://fonts/{fontstack}/{range}.pbf'; - } + .replace('{style}', path.basename(styleFile, '.json')) + .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile))) + ); + styleJSON.sprite = `local://styles/${id}/sprite`; + } + if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) { + styleJSON.glyphs = 'local://fonts/{fontstack}/{range}.pbf'; + } - repo[id] = styleJSON; - - app.get(`/${id}/style.json`, (req, res, next) => { - const fixUrl = (url, opt_nokey) => { - if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) { - return url; - } - const queryParams = []; - if (!opt_nokey && req.query.key) { - queryParams.unshift(`key=${req.query.key}`); - } - let query = ''; - if (queryParams.length) { - query = `?${queryParams.join('&')}`; - } - return url.replace( - 'local://', utils.getPublicUrl(publicUrl, req)) + query; + repo[id] = { + styleJSON, + spritePath, + publicUrl, + name: styleJSON.name }; - const styleJSON_ = clone(styleJSON); - for (const name of Object.keys(styleJSON_.sources)) { - const source = styleJSON_.sources[name]; - source.url = fixUrl(source.url); - } - // mapbox-gl-js viewer cannot handle sprite urls with query - if (styleJSON_.sprite) { - styleJSON_.sprite = fixUrl(styleJSON_.sprite, true); - } - if (styleJSON_.glyphs) { - styleJSON_.glyphs = fixUrl(styleJSON_.glyphs, false); - } - return res.send(styleJSON_); - }); - - app.get(`/${id}/sprite:scale(@[23]x)?.:format([\\w]+)`, - (req, res, next) => { - if (!spritePath) { - return res.status(404).send('File not found'); - } - const scale = req.params.scale, - format = req.params.format; - const filename = `${spritePath + (scale || '')}.${format}`; - return fs.readFile(filename, (err, data) => { - if (err) { - console.log('Sprite load error:', filename); - return res.status(404).send('File not found'); - } else { - if (format === 'json') res.header('Content-type', 'application/json'); - if (format === 'png') res.header('Content-type', 'image/png'); - return res.send(data); - } - }); - }); - - return Promise.resolve(app); + return true; + } }; diff --git a/src/server.js b/src/server.js index 9e2d070..656e01c 100644 --- a/src/server.js +++ b/src/server.js @@ -7,6 +7,7 @@ process.env.UV_THREADPOOL_SIZE = const fs = require('fs'); const path = require('path'); +const chokidar = require('chokidar'); const clone = require('clone'); const cors = require('cors'); const enableShutdown = require('http-shutdown'); @@ -103,15 +104,21 @@ function start(opts) { app.use(cors()); } - for (const id of Object.keys(config.styles || {})) { - const item = config.styles[id]; - if (!item.style || item.style.length === 0) { - console.log(`Missing "style" property for ${id}`); - continue; - } + app.use('/data/', serve_data.init(options, serving.data)); + app.use('/styles/', serve_style.init(options, serving.styles)); + if (serve_rendered) { + startupPromises.push( + serve_rendered.init(options, serving.rendered) + .then(sub => { + app.use('/styles/', sub); + }) + ); + } + let addStyle = (id, item, allowMoreData, reportFonts) => { + let success = true; if (item.serve_data !== false) { - startupPromises.push(serve_style(options, serving.styles, item, id, opts.publicUrl, + success = serve_style.add(options, serving.styles, item, id, opts.publicUrl, (mbtiles, fromData) => { let dataItemId; for (const id of Object.keys(data)) { @@ -127,44 +134,52 @@ function start(opts) { } if (dataItemId) { // mbtiles exist in the data config return dataItemId; - } else if (fromData) { - console.log(`ERROR: data "${mbtiles}" not found!`); - process.exit(1); } else { - let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles; - while (data[id]) id += '_'; - data[id] = { - 'mbtiles': mbtiles - }; - return id; + if (fromData || !allowMoreData) { + console.log(`ERROR: style "${file.name}" using unknown mbtiles "${mbtiles}"! Skipping...`); + return undefined; + } else { + let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles; + while (data[id]) id += '_'; + data[id] = { + 'mbtiles': mbtiles + }; + return id; + } } }, font => { - serving.fonts[font] = true; - }).then(sub => { - app.use('/styles/', sub); - })); + if (reportFonts) { + serving.fonts[font] = true; + } + }); } - if (item.serve_rendered !== false) { + if (success && item.serve_rendered !== false) { if (serve_rendered) { - startupPromises.push( - serve_rendered(options, serving.rendered, item, id, opts.publicUrl, - mbtiles => { - let mbtilesFile; - for (const id of Object.keys(data)) { - if (id === mbtiles) { - mbtilesFile = data[id].mbtiles; - } + startupPromises.push(serve_rendered.add(options, serving.rendered, item, id, opts.publicUrl, + mbtiles => { + let mbtilesFile; + for (const id of Object.keys(data)) { + if (id === mbtiles) { + mbtilesFile = data[id].mbtiles; } - return mbtilesFile; } - ).then(sub => { - app.use('/styles/', sub); - }) - ); + return mbtilesFile; + } + )); } else { item.serve_rendered = false; } } + }; + + for (const id of Object.keys(config.styles || {})) { + const item = config.styles[id]; + if (!item.style || item.style.length === 0) { + console.log(`Missing "style" property for ${id}`); + continue; + } + + addStyle(id, item, true, true); } startupPromises.push( @@ -181,17 +196,54 @@ function start(opts) { } startupPromises.push( - serve_data(options, serving.data, item, id, serving.styles, opts.publicUrl).then(sub => { - app.use('/data/', sub); - }) + serve_data.add(options, serving.data, item, id, opts.publicUrl) ); } + if (options.serveAllStyles) { + fs.readdir(options.paths.styles, {withFileTypes: true}, (err, files) => { + if (err) { + return; + } + for (const file of files) { + if (file.isFile() && + path.extname(file.name).toLowerCase() == '.json') { + let id = path.basename(file.name, '.json'); + let item = { + style: file.name + }; + addStyle(id, item, false, false); + } + } + }); + + const watcher = chokidar.watch(path.join(options.paths.styles, '*.json'), + { + }); + watcher.on('all', + (eventType, filename) => { + if (filename) { + let id = path.basename(filename, '.json'); + console.log(`Style "${id}" changed, updating...`); + + serve_style.remove(serving.styles, id); + serve_rendered.remove(serving.rendered, id); + + if (eventType == "add" || eventType == "change") { + let item = { + style: filename + }; + addStyle(id, item, false, false); + } + } + }); + } + app.get('/styles.json', (req, res, next) => { const result = []; const query = req.query.key ? (`?key=${req.query.key}`) : ''; for (const id of Object.keys(serving.styles)) { - const styleJSON = serving.styles[id]; + const styleJSON = serving.styles[id].styleJSON; result.push({ version: styleJSON.version, name: styleJSON.name, @@ -204,7 +256,7 @@ function start(opts) { const addTileJSONs = (arr, req, type) => { for (const id of Object.keys(serving[type])) { - const info = clone(serving[type][id]); + const info = clone(serving[type][id].tileJSON); let path = ''; if (type === 'rendered') { path = `styles/${id}`; @@ -276,14 +328,14 @@ function start(opts) { }; serveTemplate('/$', 'index', req => { - const styles = clone(config.styles || {}); + const styles = clone(serving.styles || {}); for (const id of Object.keys(styles)) { const style = styles[id]; style.name = (serving.styles[id] || serving.rendered[id] || {}).name; style.serving_data = serving.styles[id]; style.serving_rendered = serving.rendered[id]; if (style.serving_rendered) { - const center = style.serving_rendered.center; + const center = style.serving_rendered.tileJSON.center; if (center) { style.viewer_hash = `#${center[2]}/${center[1].toFixed(5)}/${center[0].toFixed(5)}`; @@ -292,8 +344,8 @@ function start(opts) { } style.xyz_link = utils.getTileUrls( - req, style.serving_rendered.tiles, - `styles/${id}`, style.serving_rendered.format, opts.publicUrl)[0]; + req, style.serving_rendered.tileJSON.tiles, + `styles/${id}`, style.serving_rendered.tileJSON.format, opts.publicUrl)[0]; } } const data = clone(serving.data || {}); @@ -337,7 +389,7 @@ function start(opts) { serveTemplate('/styles/:id/$', 'viewer', req => { const id = req.params.id; - const style = clone((config.styles || {})[id]); + const style = clone(((serving.styles || {})[id] || {}).styleJSON); if (!style) { return null; } @@ -355,7 +407,7 @@ function start(opts) { */ serveTemplate('/styles/:id/wmts.xml', 'wmts', req => { const id = req.params.id; - const wmts = clone((config.styles || {})[id]); + const wmts = clone((serving.styles || {})[id]); if (!wmts) { return null; } diff --git a/test/setup.js b/test/setup.js index 2edaab0..4e8d0a0 100644 --- a/test/setup.js +++ b/test/setup.js @@ -3,6 +3,8 @@ process.env.NODE_ENV = 'test'; global.should = require('should'); global.supertest = require('supertest'); +require = require('esm')(module); + before(function() { console.log('global setup'); process.chdir('test_data');