From a6dadfda285e9a8abdcae2a217ae325d7ca296ab Mon Sep 17 00:00:00 2001 From: Andrew Calcutt Date: Sun, 15 Oct 2023 03:03:15 -0400 Subject: [PATCH] Add pmtiles support (#1009) Adds pmtiles support to TileServer-GL Signed-off-by: Andrew Calcutt Signed-off-by: Michael Nutt Co-authored-by: Michael Nutt --- LICENSE.md | 32 ++++ docs/config.rst | 62 +++++-- package-lock.json | 54 +++++-- package.json | 2 + public/templates/index.tmpl | 5 +- src/main.js | 267 +++++++++++++++++++----------- src/pmtiles_adapter.js | 151 +++++++++++++++++ src/serve_data.js | 276 +++++++++++++++++++++---------- src/serve_rendered.js | 315 ++++++++++++++++++++++++------------ src/serve_style.js | 28 ++-- src/server.js | 176 +++++++++++++------- src/utils.js | 13 +- 12 files changed, 992 insertions(+), 389 deletions(-) create mode 100644 src/pmtiles_adapter.js diff --git a/LICENSE.md b/LICENSE.md index 74be7b0..c9126da 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1416,6 +1416,38 @@ modification, are permitted provided that the following conditions are met: this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +``` +--- + +### [PMTiles](https://github.com/protomaps/pmtiles) +``` +BSD 3-Clause License +Copyright 2021 Protomaps LLC + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE diff --git a/docs/config.rst b/docs/config.rst index fbeb35e..9cd8620 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -16,7 +16,8 @@ Example: "sprites": "sprites", "icons": "icons", "styles": "styles", - "mbtiles": "" + "mbtiles": "data", + "pmtiles": "data" }, "domains": [ "localhost:8080", @@ -180,11 +181,27 @@ Each item in this object defines one style (map). It can have the following opti ``data`` ======== -Each item specifies one data source which should be made accessible by the server. It has the following options: +Each item specifies one data source which should be made accessible by the server. It has to have one of the following options: -* ``mbtiles`` -- name of the mbtiles file [required] +* ``mbtiles`` -- name of the mbtiles file +* ``pmtiles`` -- name of the pmtiles file or url. -The mbtiles file does not need to be specified here unless you explicitly want to serve the raw data. +For example:: + + "data": { + "source1": { + "mbtiles": "source1.mbtiles" + }, + "source2": { + "pmtiles": "source2.pmtiles" + }, + "source3": { + "pmtiles": "https://foo.lan/source3.pmtiles" + } + } + + +The data source does not need to be specified here unless you explicitly want to serve the raw data. Referencing local files from style JSON ======================================= @@ -194,21 +211,46 @@ You can link various data sources from the style JSON (for example even remote T MBTiles ------- -To specify that you want to use local mbtiles, use to following syntax: ``mbtiles://switzerland.mbtiles``. -The TileServer-GL will try to find the file ``switzerland.mbtiles`` in ``root`` + ``mbtiles`` path. +To specify that you want to use local mbtiles, use to following syntax: ``mbtiles://source1.mbtiles``. +TileServer-GL will try to find the file ``source1.mbtiles`` in ``root`` + ``mbtiles`` path. For example:: "sources": { "source1": { - "url": "mbtiles://switzerland.mbtiles", + "url": "mbtiles://source1.mbtiles", "type": "vector" } } -Alternatively, you can use ``mbtiles://{zurich-vector}`` to reference existing data object from the config. -In this case, the server will look into the ``config.json`` to determine what mbtiles file to use. -For the config above, this is equivalent to ``mbtiles://zurich.mbtiles``. +Alternatively, you can use ``mbtiles://{source1}`` to reference existing data object from the config. +In this case, the server will look into the ``config.json`` to determine what file to use by data id. +For the config above, this is equivalent to ``mbtiles://source1.mbtiles``. + +PMTiles +------- + +To specify that you want to use local pmtiles, use to following syntax: ``pmtiles://source2.pmtiles``. +TileServer-GL will try to find the file ``source2.pmtiles`` in ``root`` + ``pmtiles`` path. + +To specify that you want to use a url based pmtiles, use to following syntax: ``pmtiles://https://foo.lan/source3.pmtiles``. + +For example:: + + "sources": { + "source2": { + "url": "pmtiles://source2.pmtiles", + "type": "vector" + }, + "source3": { + "url": "pmtiles://https://foo.lan/source3.pmtiles", + "type": "vector" + }, + } + +Alternatively, you can use ``pmtiles://{source2}`` to reference existing data object from the config. +In this case, the server will look into the ``config.json`` to determine what file to use by data id. +For the config above, this is equivalent to ``pmtiles://source2.mbtiles``. Sprites ------- diff --git a/package-lock.json b/package-lock.json index 05b294c..db105bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@mapbox/vector-tile": "1.3.1", "@maplibre/maplibre-gl-native": "5.2.0", "@maplibre/maplibre-gl-style-spec": "18.0.0", + "@sindresorhus/fnv1a": "3.0.0", "advanced-pool": "0.3.3", "canvas": "2.11.2", "chokidar": "3.5.3", @@ -28,6 +29,7 @@ "http-shutdown": "1.2.2", "morgan": "1.10.0", "pbf": "3.2.1", + "pmtiles": "2.11.0", "proj4": "2.9.1", "request": "2.88.2", "sanitize-filename": "1.6.3", @@ -985,6 +987,17 @@ "@octokit/openapi-types": "^12.11.0" } }, + "node_modules/@sindresorhus/fnv1a": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/fnv1a/-/fnv1a-3.0.0.tgz", + "integrity": "sha512-M6pmbdZqAryzjZ4ELAzrdCMoMZk5lH/fshKrapfSeXdf2W+GDqZvPmfXaNTZp43//FVbSwkTPwpEMnehSyskkQ==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -3251,9 +3264,9 @@ "dev": true }, "node_modules/fast-fifo": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.0.tgz", - "integrity": "sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw==" + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" }, "node_modules/fast-glob": { "version": "3.2.12", @@ -3297,6 +3310,11 @@ "reusify": "^1.0.4" } }, + "node_modules/fflate": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.1.tgz", + "integrity": "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ==" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -5964,9 +5982,9 @@ } }, "node_modules/node-addon-api": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", - "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", + "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==" }, "node_modules/node-fetch": { "version": "2.6.7", @@ -6630,6 +6648,14 @@ "node": ">=4" } }, + "node_modules/pmtiles": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pmtiles/-/pmtiles-2.11.0.tgz", + "integrity": "sha512-dU9SzzaqmCGpdEuTnIba6bDHT6j09ZJFIXxwGpvkiEnce3ZnBB1VKt6+EOmJGueriweaZLAMTUmKVElU2CBe0g==", + "dependencies": { + "fflate": "^0.8.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", @@ -7492,6 +7518,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/sharp/node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" + }, "node_modules/sharp/node_modules/simple-get": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", @@ -7862,11 +7893,6 @@ } } }, - "node_modules/sqlite3/node_modules/node-addon-api": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz", - "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==" - }, "node_modules/sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -7912,9 +7938,9 @@ } }, "node_modules/streamx": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.0.tgz", - "integrity": "sha512-HcxY6ncGjjklGs1xsP1aR71INYcsXFJet5CU1CHqihQ2J5nOsbd4OjgjHO42w/4QNv9gZb3BueV+Vxok5pLEXg==", + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz", + "integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==", "dependencies": { "fast-fifo": "^1.1.0", "queue-tick": "^1.0.1" diff --git a/package.json b/package.json index 68f8e5b..dc44a3e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "@mapbox/vector-tile": "1.3.1", "@maplibre/maplibre-gl-native": "5.2.0", "@maplibre/maplibre-gl-style-spec": "18.0.0", + "@sindresorhus/fnv1a": "3.0.0", "advanced-pool": "0.3.3", "canvas": "2.11.2", "chokidar": "3.5.3", @@ -37,6 +38,7 @@ "http-shutdown": "1.2.2", "morgan": "1.10.0", "pbf": "3.2.1", + "pmtiles": "2.11.0", "proj4": "2.9.1", "request": "2.88.2", "sanitize-filename": "1.6.3", diff --git a/public/templates/index.tmpl b/public/templates/index.tmpl index 26b6942..8708881 100644 --- a/public/templates/index.tmpl +++ b/public/templates/index.tmpl @@ -77,8 +77,9 @@ {{name}} preview {{/if}}
-

{{name}}

-

identifier: {{@key}}{{#if formatted_filesize}} | size: {{formatted_filesize}}{{/if}} | type: {{#is_vector}}vector{{/is_vector}}{{^is_vector}}raster{{/is_vector}} data

+

{{tileJSON.name}}

+
identifier: {{@key}}{{#if formatted_filesize}} | size: {{formatted_filesize}}{{/if}}
+
type: {{#is_vector}}vector{{/is_vector}}{{^is_vector}}raster{{/is_vector}} data {{#if source_type}} | ext: {{source_type}}{{/if}}

services: TileJSON {{#if wmts_link}} diff --git a/src/main.js b/src/main.js index b9a5adf..587c1a9 100644 --- a/src/main.js +++ b/src/main.js @@ -7,8 +7,9 @@ import path from 'path'; import { fileURLToPath } from 'url'; import request from 'request'; import { server } from './server.js'; - import MBTiles from '@mapbox/mbtiles'; +import { isValidHttpUrl } from './utils.js'; +import { PMtilesOpen, GetPMtilesInfo } from './pmtiles_adapter.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -25,9 +26,15 @@ import { program } from 'commander'; program .description('tileserver-gl startup options') .usage('tileserver-gl [mbtiles] [options]') + .option( + '--file ', + 'MBTiles or PMTiles file\n' + + '\t ignored if the configuration file is also specified', + ) .option( '--mbtiles ', - 'MBTiles file (uses demo configuration);\n' + + '(DEPRECIATED) MBTiles file\n' + + '\t ignored if file is also specified' + '\t ignored if the configuration file is also specified', ) .option( @@ -55,7 +62,7 @@ const opts = program.opts(); console.log(`Starting ${packageJson.name} v${packageJson.version}`); -const startServer = (configPath, config) => { +const StartServer = (configPath, config) => { let publicUrl = opts.public_url; if (publicUrl && publicUrl.lastIndexOf('/') !== publicUrl.length - 1) { publicUrl += '/'; @@ -74,135 +81,205 @@ const startServer = (configPath, config) => { }); }; -const startWithMBTiles = (mbtilesFile) => { - console.log(`[INFO] Automatically creating config file for ${mbtilesFile}`); +const StartWithInputFile = async (inputFile) => { + console.log(`[INFO] Automatically creating config file for ${inputFile}`); console.log(`[INFO] Only a basic preview style will be used.`); console.log( `[INFO] See documentation to learn how to create config.json file.`, ); - mbtilesFile = path.resolve(process.cwd(), mbtilesFile); + let inputFilePath; + if (isValidHttpUrl(inputFile)) { + inputFilePath = process.cwd(); + } else { + inputFile = path.resolve(process.cwd(), inputFile); + inputFilePath = path.dirname(inputFile); - const mbtilesStats = fs.statSync(mbtilesFile); - if (!mbtilesStats.isFile() || mbtilesStats.size === 0) { - console.log(`ERROR: Not valid MBTiles file: ${mbtilesFile}`); - process.exit(1); - } - const instance = new MBTiles(mbtilesFile + '?mode=ro', (err) => { - if (err) { - console.log('ERROR: Unable to open MBTiles.'); - console.log(`Make sure ${path.basename(mbtilesFile)} is valid MBTiles.`); + const inputFileStats = fs.statSync(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + console.log(`ERROR: Not a valid input file: `); process.exit(1); } + } - instance.getInfo((err, info) => { - if (err || !info) { - console.log('ERROR: Metadata missing in the MBTiles.'); - console.log( - `Make sure ${path.basename(mbtilesFile)} is valid MBTiles.`, - ); + const styleDir = path.resolve( + __dirname, + '../node_modules/tileserver-gl-styles/', + ); + + const config = { + options: { + paths: { + root: styleDir, + fonts: 'fonts', + styles: 'styles', + mbtiles: inputFilePath, + pmtiles: inputFilePath, + }, + }, + styles: {}, + data: {}, + }; + + const extension = inputFile.split('.').pop().toLowerCase(); + if (extension === 'pmtiles') { + let FileOpenInfo = PMtilesOpen(inputFile); + const metadata = await GetPMtilesInfo(FileOpenInfo); + + if ( + metadata.format === 'pbf' && + metadata.name.toLowerCase().indexOf('openmaptiles') > -1 + ) { + if (isValidHttpUrl(inputFile)) { + config['data'][`v3`] = { + pmtiles: inputFile, + }; + } else { + config['data'][`v3`] = { + pmtiles: path.basename(inputFile), + }; + } + + const styles = fs.readdirSync(path.resolve(styleDir, 'styles')); + for (const styleName of styles) { + const styleFileRel = styleName + '/style.json'; + const styleFile = path.resolve(styleDir, 'styles', styleFileRel); + if (fs.existsSync(styleFile)) { + config['styles'][styleName] = { + style: styleFileRel, + tilejson: { + bounds: metadata.bounds, + }, + }; + } + } + } else { + console.log( + `WARN: PMTiles not in "openmaptiles" format. Serving raw data only...`, + ); + if (isValidHttpUrl(inputFile)) { + config['data'][(metadata.id || 'pmtiles').replace(/[?/:]/g, '_')] = { + pmtiles: inputFile, + }; + } else { + config['data'][(metadata.id || 'pmtiles').replace(/[?/:]/g, '_')] = { + pmtiles: path.basename(inputFile), + }; + } + } + + if (opts.verbose) { + console.log(JSON.stringify(config, undefined, 2)); + } else { + console.log('Run with --verbose to see the config file here.'); + } + + return StartServer(null, config); + } else { + if (isValidHttpUrl(inputFile)) { + console.log( + `ERROR: MBTiles does not support web based files. "${inputFile}" is not a valid data file.`, + ); + process.exit(1); + } + const instance = new MBTiles(inputFile + '?mode=ro', (err) => { + if (err) { + console.log('ERROR: Unable to open MBTiles.'); + console.log(`Make sure ${path.basename(inputFile)} is valid MBTiles.`); process.exit(1); } - const bounds = info.bounds; - const styleDir = path.resolve( - __dirname, - '../node_modules/tileserver-gl-styles/', - ); - - const config = { - options: { - paths: { - root: styleDir, - fonts: 'fonts', - styles: 'styles', - mbtiles: path.dirname(mbtilesFile), - }, - }, - styles: {}, - data: {}, - }; - - if ( - info.format === 'pbf' && - info.name.toLowerCase().indexOf('openmaptiles') > -1 - ) { - config['data'][`v3`] = { - mbtiles: path.basename(mbtilesFile), - }; - - const styles = fs.readdirSync(path.resolve(styleDir, 'styles')); - for (const styleName of styles) { - const styleFileRel = styleName + '/style.json'; - const styleFile = path.resolve(styleDir, 'styles', styleFileRel); - if (fs.existsSync(styleFile)) { - config['styles'][styleName] = { - style: styleFileRel, - tilejson: { - bounds: bounds, - }, - }; - } + instance.getInfo((err, info) => { + if (err || !info) { + console.log('ERROR: Metadata missing in the MBTiles.'); + console.log( + `Make sure ${path.basename(inputFile)} is valid MBTiles.`, + ); + process.exit(1); } - } else { - console.log( - `WARN: MBTiles not in "openmaptiles" format. Serving raw data only...`, - ); - config['data'][ - (info.id || 'mbtiles') - .replace(/\//g, '_') - .replace(/:/g, '_') - .replace(/\?/g, '_') - ] = { - mbtiles: path.basename(mbtilesFile), - }; - } + const bounds = info.bounds; - if (opts.verbose) { - console.log(JSON.stringify(config, undefined, 2)); - } else { - console.log('Run with --verbose to see the config file here.'); - } + if ( + info.format === 'pbf' && + info.name.toLowerCase().indexOf('openmaptiles') > -1 + ) { + config['data'][`v3`] = { + mbtiles: path.basename(inputFile), + }; - return startServer(null, config); + const styles = fs.readdirSync(path.resolve(styleDir, 'styles')); + for (const styleName of styles) { + const styleFileRel = styleName + '/style.json'; + const styleFile = path.resolve(styleDir, 'styles', styleFileRel); + if (fs.existsSync(styleFile)) { + config['styles'][styleName] = { + style: styleFileRel, + tilejson: { + bounds: bounds, + }, + }; + } + } + } else { + console.log( + `WARN: MBTiles not in "openmaptiles" format. Serving raw data only...`, + ); + config['data'][(info.id || 'mbtiles').replace(/[?/:]/g, '_')] = { + mbtiles: path.basename(inputFile), + }; + } + + if (opts.verbose) { + console.log(JSON.stringify(config, undefined, 2)); + } else { + console.log('Run with --verbose to see the config file here.'); + } + + return StartServer(null, config); + }); }); - }); + } }; fs.stat(path.resolve(opts.config), (err, stats) => { if (err || !stats.isFile() || stats.size === 0) { - let mbtiles = opts.mbtiles; - if (!mbtiles) { + let inputFile; + if (opts.file) { + inputFile = opts.file; + } else if (opts.mbtiles) { + inputFile = opts.mbtiles; + } + + if (inputFile) { + return StartWithInputFile(inputFile); + } else { // try to find in the cwd const files = fs.readdirSync(process.cwd()); for (const filename of files) { - if (filename.endsWith('.mbtiles')) { - const mbTilesStats = fs.statSync(filename); - if (mbTilesStats.isFile() && mbTilesStats.size > 0) { - mbtiles = filename; + if (filename.endsWith('.mbtiles') || filename.endsWith('.pmtiles')) { + const inputFilesStats = fs.statSync(filename); + if (inputFilesStats.isFile() && inputFilesStats.size > 0) { + inputFile = filename; break; } } } - if (mbtiles) { - console.log(`No MBTiles specified, using ${mbtiles}`); - return startWithMBTiles(mbtiles); + if (inputFile) { + console.log(`No input file specified, using ${inputFile}`); + return StartWithInputFile(inputFile); } else { const url = 'https://github.com/maptiler/tileserver-gl/releases/download/v1.3.0/zurich_switzerland.mbtiles'; const filename = 'zurich_switzerland.mbtiles'; const stream = fs.createWriteStream(filename); - console.log(`No MBTiles found`); + console.log(`No input file found`); console.log(`[DEMO] Downloading sample data (${filename}) from ${url}`); - stream.on('finish', () => startWithMBTiles(filename)); + stream.on('finish', () => StartWithInputFile(filename)); return request.get(url).pipe(stream); } } - if (mbtiles) { - return startWithMBTiles(mbtiles); - } } else { console.log(`Using specified config file from ${opts.config}`); - return startServer(opts.config, null); + return StartServer(opts.config, null); } }); diff --git a/src/pmtiles_adapter.js b/src/pmtiles_adapter.js new file mode 100644 index 0000000..a827584 --- /dev/null +++ b/src/pmtiles_adapter.js @@ -0,0 +1,151 @@ +import fs from 'node:fs'; +import PMTiles from 'pmtiles'; +import { isValidHttpUrl } from './utils.js'; + +class PMTilesFileSource { + constructor(fd) { + this.fd = fd; + } + getKey() { + return this.fd; + } + async getBytes(offset, length) { + const buffer = Buffer.alloc(length); + await ReadFileBytes(this.fd, buffer, offset); + const ab = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength, + ); + return { data: ab }; + } +} + +/** + * + * @param fd + * @param buffer + * @param offset + */ +async function ReadFileBytes(fd, buffer, offset) { + return new Promise((resolve, reject) => { + fs.read(fd, buffer, 0, buffer.length, offset, (err) => { + if (err) { + return reject(err); + } + resolve(); + }); + }); +} + +/** + * + * @param FilePath + */ +export function PMtilesOpen(FilePath) { + let pmtiles = undefined; + + if (isValidHttpUrl(FilePath)) { + const source = new PMTiles.FetchSource(FilePath); + pmtiles = new PMTiles.PMTiles(source); + } else { + const fd = fs.openSync(FilePath, 'r'); + const source = new PMTilesFileSource(fd); + pmtiles = new PMTiles.PMTiles(source); + } + return pmtiles; +} + +/** + * + * @param pmtiles + */ +export async function GetPMtilesInfo(pmtiles) { + const header = await pmtiles.getHeader(); + const metadata = await pmtiles.getMetadata(); + + //Add missing metadata from header + metadata['format'] = GetPmtilesTileType(header.tileType).type; + metadata['minzoom'] = header.minZoom; + metadata['maxzoom'] = header.maxZoom; + + if (header.minLon && header.minLat && header.maxLon && header.maxLat) { + metadata['bounds'] = [ + header.minLon, + header.minLat, + header.maxLon, + header.maxLat, + ]; + } else { + metadata['bounds'] = [-180, -85.05112877980659, 180, 85.0511287798066]; + } + + if (header.centerZoom) { + metadata['center'] = [ + header.centerLon, + header.centerLat, + header.centerZoom, + ]; + } else { + metadata['center'] = [ + header.centerLon, + header.centerLat, + parseInt(metadata['maxzoom']) / 2, + ]; + } + + return metadata; +} + +/** + * + * @param pmtiles + * @param z + * @param x + * @param y + */ +export async function GetPMtilesTile(pmtiles, z, x, y) { + const header = await pmtiles.getHeader(); + const TileType = GetPmtilesTileType(header.tileType); + let zxyTile = await pmtiles.getZxy(z, x, y); + if (zxyTile && zxyTile.data) { + zxyTile = Buffer.from(zxyTile.data); + } else { + zxyTile = undefined; + } + return { data: zxyTile, header: TileType.header }; +} + +/** + * + * @param typenum + */ +function GetPmtilesTileType(typenum) { + let head = {}; + let tileType; + switch (typenum) { + case 0: + tileType = 'Unknown'; + break; + case 1: + tileType = 'pbf'; + head['Content-Type'] = 'application/x-protobuf'; + break; + case 2: + tileType = 'png'; + head['Content-Type'] = 'image/png'; + break; + case 3: + tileType = 'jpeg'; + head['Content-Type'] = 'image/jpeg'; + break; + case 4: + tileType = 'webp'; + head['Content-Type'] = 'image/webp'; + break; + case 5: + tileType = 'avif'; + head['Content-Type'] = 'image/avif'; + break; + } + return { type: tileType, header: head }; +} diff --git a/src/serve_data.js b/src/serve_data.js index 849d89a..0221369 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -10,7 +10,12 @@ import MBTiles from '@mapbox/mbtiles'; import Pbf from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; -import { getTileUrls, fixTileJSONCenter } from './utils.js'; +import { getTileUrls, isValidHttpUrl, fixTileJSONCenter } from './utils.js'; +import { + PMtilesOpen, + GetPMtilesInfo, + GetPMtilesTile, +} from './pmtiles_adapter.js'; export const serve_data = { init: (options, repo) => { @@ -18,7 +23,7 @@ export const serve_data = { app.get( '/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)', - (req, res, next) => { + async (req, res, next) => { const item = repo[req.params.id]; if (!item) { return res.sendStatus(404); @@ -48,71 +53,118 @@ export const serve_data = { ) { return res.status(404).send('Out of bounds'); } - item.source.getTile(z, x, y, (err, data, headers) => { - let isGzipped; - if (err) { - if (/does not exist/.test(err.message)) { - return res.status(204).send(); - } else { - return res - .status(500) - .header('Content-Type', 'text/plain') - .send(err.message); - } + if (item.source_type === 'pmtiles') { + let tileinfo = await GetPMtilesTile(item.source, z, x, y); + if (tileinfo == undefined || tileinfo.data == undefined) { + return res.status(404).send('Not found'); } else { - if (data == null) { - return res.status(404).send('Not found'); + let data = tileinfo.data; + let headers = tileinfo.header; + if (tileJSONFormat === 'pbf') { + if (options.dataDecoratorFunc) { + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + } + } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + + if (isGzipped) { + data = zlib.unzipSync(data); + isGzipped = false; + } + + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); + } + } + data = JSON.stringify(geojson); + } + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); + + data = zlib.gzipSync(data); + + return res.status(200).send(data); + } + } else if (item.source_type === 'mbtiles') { + item.source.getTile(z, x, y, (err, data, headers) => { + let isGzipped; + if (err) { + if (/does not exist/.test(err.message)) { + return res.status(204).send(); + } else { + return res + .status(500) + .header('Content-Type', 'text/plain') + .send(err.message); + } } else { - if (tileJSONFormat === 'pbf') { - isGzipped = - data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; - if (options.dataDecoratorFunc) { + if (data == null) { + return res.status(404).send('Not found'); + } else { + if (tileJSONFormat === 'pbf') { + isGzipped = + data.slice(0, 2).indexOf(Buffer.from([0x1f, 0x8b])) === 0; + if (options.dataDecoratorFunc) { + if (isGzipped) { + data = zlib.unzipSync(data); + isGzipped = false; + } + data = options.dataDecoratorFunc(id, 'data', data, z, x, y); + } + } + if (format === 'pbf') { + headers['Content-Type'] = 'application/x-protobuf'; + } else if (format === 'geojson') { + headers['Content-Type'] = 'application/json'; + if (isGzipped) { data = zlib.unzipSync(data); isGzipped = false; } - data = options.dataDecoratorFunc(id, 'data', data, z, x, y); - } - } - if (format === 'pbf') { - headers['Content-Type'] = 'application/x-protobuf'; - } else if (format === 'geojson') { - headers['Content-Type'] = 'application/json'; - if (isGzipped) { - data = zlib.unzipSync(data); - isGzipped = false; - } - - const tile = new VectorTile(new Pbf(data)); - const geojson = { - type: 'FeatureCollection', - features: [], - }; - for (const layerName in tile.layers) { - const layer = tile.layers[layerName]; - for (let i = 0; i < layer.length; i++) { - const feature = layer.feature(i); - const featureGeoJSON = feature.toGeoJSON(x, y, z); - featureGeoJSON.properties.layer = layerName; - geojson.features.push(featureGeoJSON); + const tile = new VectorTile(new Pbf(data)); + const geojson = { + type: 'FeatureCollection', + features: [], + }; + for (const layerName in tile.layers) { + const layer = tile.layers[layerName]; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const featureGeoJSON = feature.toGeoJSON(x, y, z); + featureGeoJSON.properties.layer = layerName; + geojson.features.push(featureGeoJSON); + } } + data = JSON.stringify(geojson); } - data = JSON.stringify(geojson); - } - delete headers['ETag']; // do not trust the tile ETag -- regenerate - headers['Content-Encoding'] = 'gzip'; - res.set(headers); + delete headers['ETag']; // do not trust the tile ETag -- regenerate + headers['Content-Encoding'] = 'gzip'; + res.set(headers); - if (!isGzipped) { - data = zlib.gzipSync(data); - isGzipped = true; - } + if (!isGzipped) { + data = zlib.gzipSync(data); + } - return res.status(200).send(data); + return res.status(200).send(data); + } } - } - }); + }); + } }, ); @@ -137,55 +189,103 @@ export const serve_data = { return app; }, - add: (options, repo, params, id, publicUrl) => { - const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles); + add: async (options, repo, params, id, publicUrl) => { + let inputFile; + let inputType; + if (params.pmtiles) { + inputType = 'pmtiles'; + if (isValidHttpUrl(params.pmtiles)) { + inputFile = params.pmtiles; + } else { + inputFile = path.resolve(options.paths.pmtiles, params.pmtiles); + } + } else if (params.mbtiles) { + inputType = 'mbtiles'; + if (isValidHttpUrl(params.mbtiles)) { + console.log( + `ERROR: MBTiles does not support web based files. "${params.mbtiles}" is not a valid data file.`, + ); + process.exit(1); + } else { + inputFile = path.resolve(options.paths.mbtiles, params.mbtiles); + } + } + let tileJSON = { tiles: params.domains || options.domains, }; - const mbtilesFileStats = fs.statSync(mbtilesFile); - if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) { - throw Error(`Not valid MBTiles file: ${mbtilesFile}`); + if (!isValidHttpUrl(inputFile)) { + const inputFileStats = fs.statSync(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + throw Error(`Not valid input file: "${inputFile}"`); + } } + let source; - const sourceInfoPromise = new Promise((resolve, reject) => { - source = new MBTiles(mbtilesFile + '?mode=ro', (err) => { - if (err) { - reject(err); - return; - } - source.getInfo((err, info) => { + let source_type; + if (inputType === 'pmtiles') { + source = PMtilesOpen(inputFile); + source_type = 'pmtiles'; + const metadata = await GetPMtilesInfo(source); + + tileJSON['name'] = id; + tileJSON['format'] = 'pbf'; + Object.assign(tileJSON, metadata); + + tileJSON['tilejson'] = '2.0.0'; + delete tileJSON['filesize']; + delete tileJSON['mtime']; + delete tileJSON['scheme']; + + Object.assign(tileJSON, params.tilejson || {}); + fixTileJSONCenter(tileJSON); + + if (options.dataDecoratorFunc) { + tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON); + } + } else if (inputType === 'mbtiles') { + source_type = 'mbtiles'; + const sourceInfoPromise = new Promise((resolve, reject) => { + source = new MBTiles(inputFile + '?mode=ro', (err) => { if (err) { reject(err); return; } - tileJSON['name'] = id; - tileJSON['format'] = 'pbf'; + source.getInfo((err, info) => { + if (err) { + reject(err); + return; + } + tileJSON['name'] = id; + tileJSON['format'] = 'pbf'; - Object.assign(tileJSON, info); + Object.assign(tileJSON, info); - tileJSON['tilejson'] = '2.0.0'; - delete tileJSON['filesize']; - delete tileJSON['mtime']; - delete tileJSON['scheme']; + tileJSON['tilejson'] = '2.0.0'; + delete tileJSON['filesize']; + delete tileJSON['mtime']; + delete tileJSON['scheme']; - Object.assign(tileJSON, params.tilejson || {}); - fixTileJSONCenter(tileJSON); + Object.assign(tileJSON, params.tilejson || {}); + fixTileJSONCenter(tileJSON); - if (options.dataDecoratorFunc) { - tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON); - } - resolve(); + if (options.dataDecoratorFunc) { + tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON); + } + resolve(); + }); }); }); - }); - return sourceInfoPromise.then(() => { - repo[id] = { - tileJSON, - publicUrl, - source, - }; - }); + await sourceInfoPromise; + } + + repo[id] = { + tileJSON, + publicUrl, + source, + source_type, + }; }, }; diff --git a/src/serve_rendered.js b/src/serve_rendered.js index c3ce22a..6968b43 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -6,7 +6,7 @@ import path from 'path'; import url from 'url'; import util from 'util'; import zlib from 'zlib'; -import sharp from 'sharp'; // sharp has to be required before node-canvas. see https://github.com/lovell/sharp/issues/371 +import sharp from 'sharp'; // sharp has to be required before node-canvas on linux but after it on windows. see https://github.com/lovell/sharp/issues/371 import { createCanvas, Image } from 'canvas'; import clone from 'clone'; import Color from 'color'; @@ -18,7 +18,17 @@ import MBTiles from '@mapbox/mbtiles'; import polyline from '@mapbox/polyline'; import proj4 from 'proj4'; import request from 'request'; -import { getFontsPbf, getTileUrls, fixTileJSONCenter } from './utils.js'; +import { + getFontsPbf, + getTileUrls, + isValidHttpUrl, + fixTileJSONCenter, +} from './utils.js'; +import { + PMtilesOpen, + GetPMtilesInfo, + GetPMtilesTile, +} from './pmtiles_adapter.js'; const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+.?\\d+)'; const PATH_PATTERN = @@ -1207,11 +1217,12 @@ export const serve_rendered = { return Promise.all([fontListingPromise]).then(() => app); }, - add: (options, repo, params, id, publicUrl, dataResolver) => { + add: async (options, repo, params, id, publicUrl, dataResolver) => { const map = { renderers: [], renderers_static: [], sources: {}, + source_types: {}, }; let styleJSON; @@ -1220,7 +1231,7 @@ export const serve_rendered = { const renderer = new mlgl.Map({ mode: mode, ratio: ratio, - request: (req, callback) => { + request: async (req, callback) => { const protocol = req.url.split(':')[0]; // console.log('Handling request:', req); if (protocol === 'sprites') { @@ -1247,17 +1258,23 @@ export const serve_rendered = { callback(err, { data: null }); }, ); - } else if (protocol === 'mbtiles') { + } else if (protocol === 'mbtiles' || protocol === 'pmtiles') { const parts = req.url.split('/'); const sourceId = parts[2]; const source = map.sources[sourceId]; + const source_type = map.source_types[sourceId]; const sourceInfo = styleJSON.sources[sourceId]; + const z = parts[3] | 0; const x = parts[4] | 0; const y = parts[5].split('.')[0] | 0; const format = parts[5].split('.')[1]; - source.getTile(z, x, y, (err, data, headers) => { - if (err) { + + if (source_type === 'pmtiles') { + let tileinfo = await GetPMtilesTile(source, z, x, y); + let data = tileinfo.data; + let headers = tileinfo.header; + if (data == undefined) { if (options.verbose) console.log('MBTiles error, serving empty', err); createEmptyResponse( @@ -1266,41 +1283,75 @@ export const serve_rendered = { callback, ); return; - } - - const response = {}; - if (headers['Last-Modified']) { - response.modified = new Date(headers['Last-Modified']); - } - - if (format === 'pbf') { - try { - response.data = zlib.unzipSync(data); - } catch (err) { - console.log( - 'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf', - id, - z, - x, - y, - ); - } - if (options.dataDecoratorFunc) { - response.data = options.dataDecoratorFunc( - sourceId, - 'data', - response.data, - z, - x, - y, - ); - } } else { + const response = {}; response.data = data; - } + if (headers['Last-Modified']) { + response.modified = new Date(headers['Last-Modified']); + } - callback(null, response); - }); + if (format === 'pbf') { + if (options.dataDecoratorFunc) { + response.data = options.dataDecoratorFunc( + sourceId, + 'data', + response.data, + z, + x, + y, + ); + } + } + + callback(null, response); + } + } else if (source_type === 'mbtiles') { + source.getTile(z, x, y, (err, data, headers) => { + if (err) { + if (options.verbose) + console.log('MBTiles error, serving empty', err); + createEmptyResponse( + sourceInfo.format, + sourceInfo.color, + callback, + ); + return; + } + + const response = {}; + if (headers['Last-Modified']) { + response.modified = new Date(headers['Last-Modified']); + } + + if (format === 'pbf') { + try { + response.data = zlib.unzipSync(data); + } catch (err) { + console.log( + 'Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf', + id, + z, + x, + y, + ); + } + if (options.dataDecoratorFunc) { + response.data = options.dataDecoratorFunc( + sourceId, + 'data', + response.data, + z, + x, + y, + ); + } + } else { + response.data = data; + } + + callback(null, response); + }); + } } else if (protocol === 'http' || protocol === 'https') { request( { @@ -1416,82 +1467,136 @@ export const serve_rendered = { const queue = []; for (const name of Object.keys(styleJSON.sources)) { + let source_type; let source = styleJSON.sources[name]; - const url = source.url; - - if (url && url.lastIndexOf('mbtiles:', 0) === 0) { - // found mbtiles source, replace with info from local file + let url = source.url; + if ( + url && + (url.startsWith('pmtiles://') || url.startsWith('mbtiles://')) + ) { + // found pmtiles or mbtiles source, replace with info from local file delete source.url; - let mbtilesFile = url.substring('mbtiles://'.length); - const fromData = - mbtilesFile[0] === '{' && mbtilesFile[mbtilesFile.length - 1] === '}'; + let dataId = url.replace('pmtiles://', '').replace('mbtiles://', ''); + if (dataId.startsWith('{') && dataId.endsWith('}')) { + dataId = dataId.slice(1, -1); + } - if (fromData) { - mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); - const mapsTo = (params.mapping || {})[mbtilesFile]; - if (mapsTo) { - mbtilesFile = mapsTo; - } - mbtilesFile = dataResolver(mbtilesFile); - if (!mbtilesFile) { - console.error(`ERROR: data "${mbtilesFile}" not found!`); - process.exit(1); + const mapsTo = (params.mapping || {})[dataId]; + if (mapsTo) { + dataId = mapsTo; + } + + let inputFile; + const DataInfo = dataResolver(dataId); + if (DataInfo.inputfile) { + inputFile = DataInfo.inputfile; + source_type = DataInfo.filetype; + } else { + console.error(`ERROR: data "${inputFile}" not found!`); + process.exit(1); + } + + if (!isValidHttpUrl(inputFile)) { + const inputFileStats = fs.statSync(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + throw Error(`Not valid PMTiles file: "${inputFile}"`); } } - queue.push( - new Promise((resolve, reject) => { - mbtilesFile = path.resolve(options.paths.mbtiles, mbtilesFile); - const mbtilesFileStats = fs.statSync(mbtilesFile); - if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) { - throw Error(`Not valid MBTiles file: ${mbtilesFile}`); + if (source_type === 'pmtiles') { + map.sources[name] = PMtilesOpen(inputFile); + map.source_types[name] = 'pmtiles'; + const metadata = await GetPMtilesInfo(map.sources[name]); + + if (!repoobj.dataProjWGStoInternalWGS && metadata.proj4) { + // how to do this for multiple sources with different proj4 defs? + const to3857 = proj4('EPSG:3857'); + const toDataProj = proj4(metadata.proj4); + repoobj.dataProjWGStoInternalWGS = (xy) => + to3857.inverse(toDataProj.forward(xy)); + } + + const type = source.type; + Object.assign(source, metadata); + source.type = type; + source.tiles = [ + // meta url which will be detected when requested + `pmtiles://${name}/{z}/{x}/{y}.${metadata.format || 'pbf'}`, + ]; + delete source.scheme; + + if ( + !attributionOverride && + source.attribution && + source.attribution.length > 0 + ) { + if (!tileJSON.attribution.includes(source.attribution)) { + if (tileJSON.attribution.length > 0) { + tileJSON.attribution += ' | '; + } + tileJSON.attribution += source.attribution; } - map.sources[name] = new MBTiles(mbtilesFile + '?mode=ro', (err) => { - map.sources[name].getInfo((err, info) => { - if (err) { - console.error(err); - return; - } - - if (!repoobj.dataProjWGStoInternalWGS && info.proj4) { - // how to do this for multiple sources with different proj4 defs? - const to3857 = proj4('EPSG:3857'); - const toDataProj = proj4(info.proj4); - repoobj.dataProjWGStoInternalWGS = (xy) => - to3857.inverse(toDataProj.forward(xy)); - } - - const type = source.type; - Object.assign(source, info); - source.type = type; - source.tiles = [ - // meta url which will be detected when requested - `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`, - ]; - delete source.scheme; - - if (options.dataDecoratorFunc) { - source = options.dataDecoratorFunc(name, 'tilejson', source); - } - - if ( - !attributionOverride && - source.attribution && - source.attribution.length > 0 - ) { - if (!tileJSON.attribution.includes(source.attribution)) { - if (tileJSON.attribution.length > 0) { - tileJSON.attribution += ' | '; - } - tileJSON.attribution += source.attribution; + } + } else { + queue.push( + new Promise((resolve, reject) => { + inputFile = path.resolve(options.paths.mbtiles, inputFile); + const inputFileStats = fs.statSync(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + throw Error(`Not valid MBTiles file: "${inputFile}"`); + } + map.sources[name] = new MBTiles(inputFile + '?mode=ro', (err) => { + map.sources[name].getInfo((err, info) => { + if (err) { + console.error(err); + return; } - } - resolve(); + map.source_types[name] = 'mbtiles'; + + if (!repoobj.dataProjWGStoInternalWGS && info.proj4) { + // how to do this for multiple sources with different proj4 defs? + const to3857 = proj4('EPSG:3857'); + const toDataProj = proj4(info.proj4); + repoobj.dataProjWGStoInternalWGS = (xy) => + to3857.inverse(toDataProj.forward(xy)); + } + + const type = source.type; + Object.assign(source, info); + source.type = type; + source.tiles = [ + // meta url which will be detected when requested + `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`, + ]; + delete source.scheme; + + if (options.dataDecoratorFunc) { + source = options.dataDecoratorFunc( + name, + 'tilejson', + source, + ); + } + + if ( + !attributionOverride && + source.attribution && + source.attribution.length > 0 + ) { + if (!tileJSON.attribution.includes(source.attribution)) { + if (tileJSON.attribution.length > 0) { + tileJSON.attribution += ' | '; + } + tileJSON.attribution += source.attribution; + } + } + resolve(); + }); }); - }); - }), - ); + }), + ); + } } } diff --git a/src/serve_style.js b/src/serve_style.js index c8b7265..5f4e5ee 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -110,20 +110,24 @@ export const serve_style = { for (const name of Object.keys(styleJSON.sources)) { const source = styleJSON.sources[name]; - const url = source.url; - if (url && url.lastIndexOf('mbtiles:', 0) === 0) { - let mbtilesFile = url.substring('mbtiles://'.length); - const fromData = - mbtilesFile[0] === '{' && mbtilesFile[mbtilesFile.length - 1] === '}'; + let url = source.url; + if ( + url && + (url.startsWith('pmtiles://') || url.startsWith('mbtiles://')) + ) { + const protocol = url.split(':')[0]; - if (fromData) { - mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2); - const mapsTo = (params.mapping || {})[mbtilesFile]; - if (mapsTo) { - mbtilesFile = mapsTo; - } + let dataId = url.replace('pmtiles://', '').replace('mbtiles://', ''); + if (dataId.startsWith('{') && dataId.endsWith('}')) { + dataId = dataId.slice(1, -1); } - const identifier = reportTiles(mbtilesFile, fromData); + + const mapsTo = (params.mapping || {})[dataId]; + if (mapsTo) { + dataId = mapsTo; + } + + const identifier = reportTiles(dataId, protocol); if (!identifier) { return false; } diff --git a/src/server.js b/src/server.js index 88d026c..0129629 100644 --- a/src/server.js +++ b/src/server.js @@ -6,7 +6,7 @@ process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, os.cpus().length * 1.5)); import fs from 'node:fs'; import path from 'path'; - +import fnv1a from '@sindresorhus/fnv1a'; import chokidar from 'chokidar'; import clone from 'clone'; import cors from 'cors'; @@ -19,7 +19,7 @@ import morgan from 'morgan'; import { serve_data } from './serve_data.js'; import { serve_style } from './serve_style.js'; import { serve_font } from './serve_font.js'; -import { getTileUrls, getPublicUrl } from './utils.js'; +import { getTileUrls, getPublicUrl, isValidHttpUrl } from './utils.js'; import { fileURLToPath } from 'url'; const __filename = fileURLToPath(import.meta.url); @@ -93,6 +93,7 @@ function start(opts) { paths.fonts = path.resolve(paths.root, paths.fonts || ''); paths.sprites = path.resolve(paths.root, paths.sprites || ''); paths.mbtiles = path.resolve(paths.root, paths.mbtiles || ''); + paths.pmtiles = path.resolve(paths.root, paths.pmtiles || ''); paths.icons = path.resolve(paths.root, paths.icons || ''); const startupPromises = []; @@ -109,6 +110,7 @@ function start(opts) { checkPath('fonts'); checkPath('sprites'); checkPath('mbtiles'); + checkPath('pmtiles'); checkPath('icons'); /** @@ -181,34 +183,43 @@ function start(opts) { item, id, opts.publicUrl, - (mbtiles, fromData) => { + (StyleSourceId, protocol) => { let dataItemId; for (const id of Object.keys(data)) { - if (fromData) { - if (id === mbtiles) { - dataItemId = id; - } + if (id === StyleSourceId) { + // Style id was found in data ids, return that id + dataItemId = id; } else { - if (data[id].mbtiles === mbtiles) { + const fileType = Object.keys(data[id])[0]; + if (data[id][fileType] === StyleSourceId) { + // Style id was found in data filename, return the id that filename belong to dataItemId = id; } } } if (dataItemId) { - // mbtiles exist in the data config + // input files exists in the data config, return found id return dataItemId; } else { - if (fromData || !allowMoreData) { + if (!allowMoreData) { console.log( - `ERROR: style "${item.style}" using unknown mbtiles "${mbtiles}"! Skipping...`, + `ERROR: style "${item.style}" using unknown file "${StyleSourceId}"! Skipping...`, ); return undefined; } else { - let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles; - while (data[id]) id += '_'; + let id = + StyleSourceId.substr(0, StyleSourceId.lastIndexOf('.')) || + StyleSourceId; + if (isValidHttpUrl(StyleSourceId)) { + id = + fnv1a(StyleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1'); + } + while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't + //Add the new data source to the data array. data[id] = { - mbtiles: mbtiles, + [protocol]: StyleSourceId, }; + return id; } } @@ -229,14 +240,24 @@ function start(opts) { item, id, opts.publicUrl, - (mbtiles) => { - let mbtilesFile; + (StyleSourceId) => { + let fileType; + let inputFile; for (const id of Object.keys(data)) { - if (id === mbtiles) { - mbtilesFile = data[id].mbtiles; + fileType = Object.keys(data[id])[0]; + if (StyleSourceId == id) { + inputFile = data[id][fileType]; + break; + } else if (data[id][fileType] == StyleSourceId) { + inputFile = data[id][fileType]; + break; } } - return mbtilesFile; + if (!isValidHttpUrl(inputFile)) { + inputFile = path.resolve(options.paths[fileType], inputFile); + } + + return { inputfile: inputFile, filetype: fileType }; }, ), ); @@ -264,8 +285,11 @@ function start(opts) { for (const id of Object.keys(data)) { const item = data[id]; - if (!item.mbtiles || item.mbtiles.length === 0) { - console.log(`Missing "mbtiles" property for ${id}`); + const fileType = Object.keys(data[id])[0]; + if (!fileType || !(fileType === 'pmtiles' || fileType === 'mbtiles')) { + console.log( + `Missing "pmtiles" or "mbtiles" property for ${id} data source`, + ); continue; } @@ -424,14 +448,16 @@ function start(opts) { }; serveTemplate('/$', 'index', (req) => { - const styles = clone(serving.styles || {}); - for (const id of Object.keys(styles)) { - const style = styles[id]; - style.name = (serving.styles[id] || serving.rendered[id] || {}).name; - style.serving_data = serving.styles[id]; - style.serving_rendered = serving.rendered[id]; + let styles = {}; + for (const id of Object.keys(serving.styles || {})) { + let style = { + ...serving.styles[id], + serving_data: serving.styles[id], + serving_rendered: serving.rendered[id], + }; + if (style.serving_rendered) { - const center = style.serving_rendered.tileJSON.center; + const { center } = style.serving_rendered.tileJSON; if (center) { style.viewer_hash = `#${center[2]}/${center[1].toFixed( 5, @@ -451,40 +477,46 @@ function start(opts) { opts.publicUrl, )[0]; } + + styles[id] = style; } - const data = clone(serving.data || {}); - for (const id of Object.keys(data)) { - const data_ = data[id]; - const tilejson = data[id].tileJSON; - const center = tilejson.center; + + let datas = {}; + for (const id of Object.keys(serving.data || {})) { + let data = Object.assign({}, serving.data[id]); + + const { tileJSON } = serving.data[id]; + const { center } = tileJSON; + if (center) { - data_.viewer_hash = `#${center[2]}/${center[1].toFixed( + data.viewer_hash = `#${center[2]}/${center[1].toFixed( 5, )}/${center[0].toFixed(5)}`; } - data_.is_vector = tilejson.format === 'pbf'; - if (!data_.is_vector) { + + data.is_vector = tileJSON.format === 'pbf'; + if (!data.is_vector) { if (center) { const centerPx = mercator.px([center[0], center[1]], center[2]); - data_.thumbnail = `${center[2]}/${Math.floor( + data.thumbnail = `${center[2]}/${Math.floor( centerPx[0] / 256, - )}/${Math.floor(centerPx[1] / 256)}.${data_.tileJSON.format}`; + )}/${Math.floor(centerPx[1] / 256)}.${tileJSON.format}`; } - data_.xyz_link = getTileUrls( + data.xyz_link = getTileUrls( req, - tilejson.tiles, + tileJSON.tiles, `data/${id}`, - tilejson.format, + tileJSON.format, opts.publicUrl, { pbf: options.pbfAlias, }, )[0]; } - if (data_.filesize) { + if (data.filesize) { let suffix = 'kB'; - let size = parseInt(data_.filesize, 10) / 1024; + let size = parseInt(tileJSON.filesize, 10) / 1024; if (size > 1024) { suffix = 'MB'; size /= 1024; @@ -493,26 +525,33 @@ function start(opts) { suffix = 'GB'; size /= 1024; } - data_.formatted_filesize = `${size.toFixed(2)} ${suffix}`; + data.formatted_filesize = `${size.toFixed(2)} ${suffix}`; } + + datas[id] = data; } + return { styles: Object.keys(styles).length ? styles : null, - data: Object.keys(data).length ? data : null, + data: Object.keys(datas).length ? datas : null, }; }); serveTemplate('/styles/:id/$', 'viewer', (req) => { - const id = req.params.id; + const { id } = req.params; const style = clone(((serving.styles || {})[id] || {}).styleJSON); + if (!style) { return null; } - style.id = id; - style.name = (serving.styles[id] || serving.rendered[id]).name; - style.serving_data = serving.styles[id]; - style.serving_rendered = serving.rendered[id]; - return style; + + return { + id, + name: (serving.styles[id] || serving.rendered[id]).name, + serving_data: serving.styles[id], + serving_rendered: serving.rendered[id], + ...style, + }; }); /* @@ -521,37 +560,49 @@ function start(opts) { }); */ serveTemplate('/styles/:id/wmts.xml', 'wmts', (req) => { - const id = req.params.id; + const { id } = req.params; const wmts = clone((serving.styles || {})[id]); + if (!wmts) { return null; } + if (wmts.hasOwnProperty('serve_rendered') && !wmts.serve_rendered) { return null; } - wmts.id = id; - wmts.name = (serving.styles[id] || serving.rendered[id]).name; + + let baseUrl; if (opts.publicUrl) { - wmts.baseUrl = opts.publicUrl; + baseUrl = opts.publicUrl; } else { - wmts.baseUrl = `${ + baseUrl = `${ req.get('X-Forwarded-Protocol') ? req.get('X-Forwarded-Protocol') : req.protocol }://${req.get('host')}/`; } - return wmts; + + return { + id, + name: (serving.styles[id] || serving.rendered[id]).name, + baseUrl, + ...wmts, + }; }); serveTemplate('/data/:id/$', 'data', (req) => { - const id = req.params.id; - const data = clone(serving.data[id]); + const { id } = req.params; + const data = serving.data[id]; + if (!data) { return null; } - data.id = id; - data.is_vector = data.tileJSON.format === 'pbf'; - return data; + + return { + id, + is_vector: data.tileJSON.format === 'pbf', + ...data, + }; }); let startupComplete = false; @@ -559,6 +610,7 @@ function start(opts) { console.log('Startup complete'); startupComplete = true; }); + app.get('/health', (req, res, next) => { if (startupComplete) { return res.status(200).send('OK'); diff --git a/src/utils.js b/src/utils.js index 6fe22f0..123fed6 100644 --- a/src/utils.js +++ b/src/utils.js @@ -2,7 +2,6 @@ import path from 'path'; import fs from 'node:fs'; - import clone from 'clone'; import glyphCompose from '@mapbox/glyph-pbf-composite'; @@ -163,3 +162,15 @@ export const getFontsPbf = ( return Promise.all(queue).then((values) => glyphCompose.combine(values)); }; + +export const isValidHttpUrl = (string) => { + let url; + + try { + url = new URL(string); + } catch (_) { + return false; + } + + return url.protocol === 'http:' || url.protocol === 'https:'; +};