From 9b673638e3d3e50ed95c36d14c55976ceaadf9dd Mon Sep 17 00:00:00 2001 From: Joseph Canero Date: Tue, 3 Oct 2017 15:30:37 -0400 Subject: [PATCH] when serving styles, look in a whitelisted set of urls to also see when we can add the key parameter to the url. also allow configuration of the parameter name. --- src/serve_style.js | 107 ++++++++++++++++++++------------ src/server.js | 149 +++++++++++++++++++++++---------------------- 2 files changed, 146 insertions(+), 110 deletions(-) diff --git a/src/serve_style.js b/src/serve_style.js index cd9dfe9..5f196a0 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -1,25 +1,27 @@ 'use strict'; var path = require('path'), - fs = require('fs'); + fs = require('fs'), + nodeUrl = require('url'), + querystring = require('querystring'); var clone = require('clone'), - express = require('express'); + express = require('express'); -module.exports = function(options, repo, params, id, reportTiles, reportFont) { +module.exports = function (options, repo, params, id, reportTiles, reportFont) { var app = express().disable('x-powered-by'); var styleFile = path.resolve(options.paths.styles, params.style); var styleJSON = clone(require(styleFile)); - Object.keys(styleJSON.sources).forEach(function(name) { + Object.keys(styleJSON.sources).forEach(function (name) { var source = styleJSON.sources[name]; var url = source.url; if (url && url.lastIndexOf('mbtiles:', 0) === 0) { var mbtilesFile = url.substring('mbtiles://'.length); var fromData = mbtilesFile[0] == '{' && - mbtilesFile[mbtilesFile.length - 1] == '}'; + mbtilesFile[mbtilesFile.length - 1] == '}'; if (fromData) { mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); @@ -33,7 +35,7 @@ module.exports = function(options, repo, params, id, reportTiles, reportFont) { } }); - styleJSON.layers.forEach(function(obj) { + styleJSON.layers.forEach(function (obj) { if (obj['type'] == 'symbol') { var fonts = (obj['layout'] || {})['text-font']; if (fonts && fonts.length) { @@ -50,10 +52,10 @@ module.exports = function(options, repo, params, id, reportTiles, reportFont) { var httpTester = /^(http(s)?:)?\/\//; if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) { spritePath = path.join(options.paths.sprites, - styleJSON.sprite - .replace('{style}', path.basename(styleFile, '.json')) - .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile))) - ); + styleJSON.sprite + .replace('{style}', path.basename(styleFile, '.json')) + .replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile))) + ); styleJSON.sprite = 'local://styles/' + id + '/sprite'; } if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) { @@ -62,28 +64,57 @@ module.exports = function(options, repo, params, id, reportTiles, reportFont) { repo[id] = styleJSON; - app.get('/' + id + '/style.json', function(req, res, next) { - var fixUrl = function(url, opt_nokey, opt_nostyle) { - if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) { + var isWhitelistedUrl = function (url) { + if (!options.auth || !Array.isArray(options.auth.keyDomains) || options.auth.keyDomains.length === 0) { + return false; + } + + for (var i = 0; i < options.auth.keyDomains.length; i++) { + var keyDomain = options.auth.keyDomains[i]; + if (!keyDomain || (typeof keyDomain !== 'string') || keyDomain.length === 0) { + continue; + } + + if (url.indexOf(keyDomain) === 0) { + return true; + } + } + + return false; + } + + app.get('/' + id + '/style.json', function (req, res, next) { + var fixUrl = function (url, opt_nokey, opt_nostyle) { + if (!url || (typeof url !== 'string') || (url.indexOf('local://') !== 0 && !isWhitelistedUrl(url))) { return url; } - var queryParams = []; + + var queryParams = {}; if (!opt_nostyle && global.addStyleParam) { - queryParams.push('style=' + id); + queryParams.style = id; } - if (!opt_nokey && req.query.key) { - queryParams.unshift('key=' + req.query.key); + if (!opt_nokey && req.query[options.auth.keyName]) { + queryParams[options.auth.keyName] = req.query[options.auth.keyName]; } - var query = ''; - if (queryParams.length) { - query = '?' + queryParams.join('&'); - } - return url.replace( + + if (url.indexOf('local://') === 0) { + var query = querystring.stringify(queryParams); + if (query.length) { + query = '?' + query; + } + return url.replace( 'local://', req.protocol + '://' + req.headers.host + '/') + query; + } else { // whitelisted url. might have existing parameters + var parsedUrl = nodeUrl.parse(url); + var parsedQS = querystring.parse(url.query); + var newParams = Object.assign(parsedQS, queryParams); + parsedUrl.search = querystring.stringify(parsedQS); + return url.format(parsedUrl); + } }; var styleJSON_ = clone(styleJSON); - Object.keys(styleJSON_.sources).forEach(function(name) { + Object.keys(styleJSON_.sources).forEach(function (name) { var source = styleJSON_.sources[name]; source.url = fixUrl(source.url); }); @@ -98,24 +129,24 @@ module.exports = function(options, repo, params, id, reportTiles, reportFont) { }); app.get('/' + id + '/sprite:scale(@[23]x)?\.:format([\\w]+)', - function(req, res, next) { - if (!spritePath) { - return res.status(404).send('File not found'); - } - var scale = req.params.scale, - format = req.params.format; - var filename = spritePath + (scale || '') + '.' + format; - return fs.readFile(filename, function(err, data) { - if (err) { - console.log('Sprite load error:', filename); + function (req, res, next) { + if (!spritePath) { return res.status(404).send('File not found'); - } else { - if (format == 'json') res.header('Content-type', 'application/json'); - if (format == 'png') res.header('Content-type', 'image/png'); - return res.send(data); } + var scale = req.params.scale, + format = req.params.format; + var filename = spritePath + (scale || '') + '.' + format; + return fs.readFile(filename, function (err, data) { + if (err) { + console.log('Sprite load error:', filename); + return res.status(404).send('File not found'); + } else { + if (format == 'json') res.header('Content-type', 'application/json'); + if (format == 'png') res.header('Content-type', 'image/png'); + return res.send(data); + } + }); }); - }); return Promise.resolve(app); }; diff --git a/src/server.js b/src/server.js index 06164ee..96731c8 100644 --- a/src/server.js +++ b/src/server.js @@ -2,26 +2,26 @@ 'use strict'; process.env.UV_THREADPOOL_SIZE = - Math.ceil(Math.max(4, require('os').cpus().length * 1.5)); + Math.ceil(Math.max(4, require('os').cpus().length * 1.5)); var fs = require('fs'), - path = require('path'); + path = require('path'); var base64url = require('base64url'), - clone = require('clone'), - cors = require('cors'), - enableShutdown = require('http-shutdown'), - express = require('express'), - handlebars = require('handlebars'), - mercator = new (require('@mapbox/sphericalmercator'))(), - morgan = require('morgan'); + clone = require('clone'), + cors = require('cors'), + enableShutdown = require('http-shutdown'), + express = require('express'), + handlebars = require('handlebars'), + mercator = new (require('@mapbox/sphericalmercator'))(), + morgan = require('morgan'); var packageJson = require('../package'), - serve_font = require('./serve_font'), - serve_rendered = null, - serve_style = require('./serve_style'), - serve_data = require('./serve_data'), - utils = require('./utils'); + serve_font = require('./serve_font'), + serve_rendered = null, + serve_style = require('./serve_style'), + serve_data = require('./serve_data'), + utils = require('./utils'); var isLight = packageJson.name.slice(-6) == '-light'; if (!isLight) { @@ -33,12 +33,12 @@ function start(opts) { console.log('Starting server'); var app = express().disable('x-powered-by'), - serving = { - styles: {}, - rendered: {}, - data: {}, - fonts: {} - }; + serving = { + styles: {}, + rendered: {}, + data: {}, + fonts: {} + }; app.enable('trust proxy'); @@ -66,6 +66,11 @@ function start(opts) { } var options = config.options || {}; + + options.auth = options.auth || {}; + options.auth.keyName = options.auth.keyName || 'key'; + options.auth.keyDomains = options.auth.keyDomains || []; + var paths = options.paths || {}; options.paths = paths; paths.root = path.resolve( @@ -78,7 +83,7 @@ function start(opts) { var startupPromises = []; - var checkPath = function(type) { + var checkPath = function (type) { if (!fs.existsSync(paths[type])) { console.error('The specified path for "' + type + '" does not exist (' + paths[type] + ').'); process.exit(1); @@ -92,7 +97,7 @@ function start(opts) { if (options.dataDecorator) { try { options.dataDecoratorFunc = require(path.resolve(paths.root, options.dataDecorator)); - } catch (e) {} + } catch (e) { } } var data = clone(config.data || {}); @@ -101,7 +106,7 @@ function start(opts) { app.use(cors()); } - Object.keys(config.styles || {}).forEach(function(id) { + Object.keys(config.styles || {}).forEach(function (id) { var item = config.styles[id]; if (!item.style || item.style.length == 0) { console.log('Missing "style" property for ' + id); @@ -110,9 +115,9 @@ function start(opts) { if (item.serve_data !== false) { startupPromises.push(serve_style(options, serving.styles, item, id, - function(mbtiles, fromData) { + function (mbtiles, fromData) { var dataItemId; - Object.keys(data).forEach(function(id) { + Object.keys(data).forEach(function (id) { if (fromData) { if (id == mbtiles) { dataItemId = id; @@ -136,9 +141,9 @@ function start(opts) { }; return id; } - }, function(font) { + }, function (font) { serving.fonts[font] = true; - }).then(function(sub) { + }).then(function (sub) { app.use('/styles/', sub); })); } @@ -146,16 +151,16 @@ function start(opts) { if (serve_rendered) { startupPromises.push( serve_rendered(options, serving.rendered, item, id, - function(mbtiles) { + function (mbtiles) { var mbtilesFile; - Object.keys(data).forEach(function(id) { + Object.keys(data).forEach(function (id) { if (id == mbtiles) { mbtilesFile = data[id].mbtiles; } }); return mbtilesFile; } - ).then(function(sub) { + ).then(function (sub) { app.use('/styles/', sub); }) ); @@ -166,12 +171,12 @@ function start(opts) { }); startupPromises.push( - serve_font(options, serving.fonts).then(function(sub) { + serve_font(options, serving.fonts).then(function (sub) { app.use('/', sub); }) ); - Object.keys(data).forEach(function(id) { + Object.keys(data).forEach(function (id) { var item = data[id]; if (!item.mbtiles || item.mbtiles.length == 0) { console.log('Missing "mbtiles" property for ' + id); @@ -179,30 +184,30 @@ function start(opts) { } startupPromises.push( - serve_data(options, serving.data, item, id, serving.styles).then(function(sub) { + serve_data(options, serving.data, item, id, serving.styles).then(function (sub) { app.use('/data/', sub); }) ); }); - app.get('/styles.json', function(req, res, next) { + app.get('/styles.json', function (req, res, next) { var result = []; var query = req.query.key ? ('?key=' + req.query.key) : ''; - Object.keys(serving.styles).forEach(function(id) { + Object.keys(serving.styles).forEach(function (id) { var styleJSON = serving.styles[id]; result.push({ version: styleJSON.version, name: styleJSON.name, id: id, url: req.protocol + '://' + req.headers.host + - '/styles/' + id + '/style.json' + query + '/styles/' + id + '/style.json' + query }); }); res.send(result); }); - var addTileJSONs = function(arr, req, type) { - Object.keys(serving[type]).forEach(function(id) { + var addTileJSONs = function (arr, req, type) { + Object.keys(serving[type]).forEach(function (id) { var info = clone(serving[type][id]); var path = ''; if (type == 'rendered') { @@ -218,13 +223,13 @@ function start(opts) { return arr; }; - app.get('/rendered.json', function(req, res, next) { + app.get('/rendered.json', function (req, res, next) { res.send(addTileJSONs([], req, 'rendered')); }); - app.get('/data.json', function(req, res, next) { + app.get('/data.json', function (req, res, next) { res.send(addTileJSONs([], req, 'data')); }); - app.get('/index.json', function(req, res, next) { + app.get('/index.json', function (req, res, next) { res.send(addTileJSONs(addTileJSONs([], req, 'rendered'), req, 'data')); }); @@ -233,25 +238,25 @@ function start(opts) { app.use('/', express.static(path.join(__dirname, '../public/resources'))); var templates = path.join(__dirname, '../public/templates'); - var serveTemplate = function(urlPath, template, dataGetter) { + var serveTemplate = function (urlPath, template, dataGetter) { var templateFile = templates + '/' + template + '.tmpl'; if (template == 'index') { if (options.frontPage === false) { return; } else if (options.frontPage && - options.frontPage.constructor === String) { + options.frontPage.constructor === String) { templateFile = path.resolve(paths.root, options.frontPage); } } - startupPromises.push(new Promise(function(resolve, reject) { - fs.readFile(templateFile, function(err, content) { + startupPromises.push(new Promise(function (resolve, reject) { + fs.readFile(templateFile, function (err, content) { if (err) { console.error('Template not found:', err); reject(err); } var compiled = handlebars.compile(content.toString()); - app.use(urlPath, function(req, res, next) { + app.use(urlPath, function (req, res, next) { var data = {}; if (dataGetter) { data = dataGetter(req); @@ -262,7 +267,7 @@ function start(opts) { data['server_version'] = packageJson.name + ' v' + packageJson.version; data['is_light'] = isLight; data['key_query_part'] = - req.query.key ? 'key=' + req.query.key + '&' : ''; + req.query.key ? 'key=' + req.query.key + '&' : ''; data['key_query'] = req.query.key ? '?key=' + req.query.key : ''; return res.status(200).send(compiled(data)); }); @@ -271,9 +276,9 @@ function start(opts) { })); }; - serveTemplate('/$', 'index', function(req) { + serveTemplate('/$', 'index', function (req) { var styles = clone(config.styles || {}); - Object.keys(styles).forEach(function(id) { + Object.keys(styles).forEach(function (id) { var style = styles[id]; style.name = (serving.styles[id] || serving.rendered[id] || {}).name; style.serving_data = serving.styles[id]; @@ -282,13 +287,13 @@ function start(opts) { var center = style.serving_rendered.center; if (center) { style.viewer_hash = '#' + center[2] + '/' + - center[1].toFixed(5) + '/' + - center[0].toFixed(5); + center[1].toFixed(5) + '/' + + center[0].toFixed(5); var centerPx = mercator.px([center[0], center[1]], center[2]); style.thumbnail = center[2] + '/' + - Math.floor(centerPx[0] / 256) + '/' + - Math.floor(centerPx[1] / 256) + '.png'; + Math.floor(centerPx[0] / 256) + '/' + + Math.floor(centerPx[1] / 256) + '.png'; } var query = req.query.key ? ('?key=' + req.query.key) : ''; @@ -297,27 +302,27 @@ function start(opts) { '/styles/' + id + '.json' + query) + '/wmts'; var tiles = utils.getTileUrls( - req, style.serving_rendered.tiles, - 'styles/' + id, style.serving_rendered.format); + req, style.serving_rendered.tiles, + 'styles/' + id, style.serving_rendered.format); style.xyz_link = tiles[0]; } }); var data = clone(serving.data || {}); - Object.keys(data).forEach(function(id) { + Object.keys(data).forEach(function (id) { var data_ = data[id]; var center = data_.center; if (center) { data_.viewer_hash = '#' + center[2] + '/' + - center[1].toFixed(5) + '/' + - center[0].toFixed(5); + center[1].toFixed(5) + '/' + + center[0].toFixed(5); } data_.is_vector = data_.format == 'pbf'; if (!data_.is_vector) { if (center) { var centerPx = mercator.px([center[0], center[1]], center[2]); data_.thumbnail = center[2] + '/' + - Math.floor(centerPx[0] / 256) + '/' + - Math.floor(centerPx[1] / 256) + '.' + data_.format; + Math.floor(centerPx[0] / 256) + '/' + + Math.floor(centerPx[1] / 256) + '.' + data_.format; } var query = req.query.key ? ('?key=' + req.query.key) : ''; @@ -326,9 +331,9 @@ function start(opts) { '/data/' + id + '.json' + query) + '/wmts'; var tiles = utils.getTileUrls( - req, data_.tiles, 'data/' + id, data_.format, { - 'pbf': options.pbfAlias - }); + req, data_.tiles, 'data/' + id, data_.format, { + 'pbf': options.pbfAlias + }); data_.xyz_link = tiles[0]; } if (data_.filesize) { @@ -351,7 +356,7 @@ function start(opts) { }; }); - serveTemplate('/styles/:id/$', 'viewer', function(req) { + serveTemplate('/styles/:id/$', 'viewer', function (req) { var id = req.params.id; var style = clone((config.styles || {})[id]); if (!style) { @@ -370,7 +375,7 @@ function start(opts) { }); */ - serveTemplate('/data/:id/$', 'data', function(req) { + serveTemplate('/data/:id/$', 'data', function (req) { var id = req.params.id; var data = clone(serving.data[id]); if (!data) { @@ -382,11 +387,11 @@ function start(opts) { }); var startupComplete = false; - var startupPromise = Promise.all(startupPromises).then(function() { + var startupPromise = Promise.all(startupPromises).then(function () { console.log('Startup complete'); startupComplete = true; }); - app.get('/health', function(req, res, next) { + app.get('/health', function (req, res, next) { if (startupComplete) { return res.status(200).send('OK'); } else { @@ -394,7 +399,7 @@ function start(opts) { } }); - var server = app.listen(process.env.PORT || opts.port, process.env.BIND || opts.bind, function() { + var server = app.listen(process.env.PORT || opts.port, process.env.BIND || opts.bind, function () { var address = this.address().address; if (address.indexOf('::') === 0) { address = '[' + address + ']'; // literal IPv6 address @@ -412,17 +417,17 @@ function start(opts) { }; } -module.exports = function(opts) { +module.exports = function (opts) { var running = start(opts); - process.on('SIGINT', function() { + process.on('SIGINT', function () { process.exit(); }); - process.on('SIGHUP', function() { + process.on('SIGHUP', function () { console.log('Stopping server and reloading config'); - running.server.shutdown(function() { + running.server.shutdown(function () { for (var key in require.cache) { delete require.cache[key]; }