Add pmtiles support (#1009)
Adds pmtiles support to TileServer-GL Signed-off-by: Andrew Calcutt <acalcutt@techidiots.net> Signed-off-by: Michael Nutt <michael@nuttnet.net> Co-authored-by: Michael Nutt <michael@nuttnet.net>
This commit is contained in:
Andrew Calcutt
GitHub
parent
7d8a6ad338
commit
a6dadfda28
12 changed files with 992 additions and 389 deletions
32
LICENSE.md
32
LICENSE.md
|
|
@ -1427,3 +1427,35 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
```
|
||||
---
|
||||
|
||||
### [PMTiles](https://github.com/protomaps/pmtiles)
|
||||
```
|
||||
BSD 3-Clause License
|
||||
Copyright 2021 Protomaps LLC
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
```
|
||||
|
|
@ -16,7 +16,8 @@ Example:
|
|||
"sprites": "sprites",
|
||||
"icons": "icons",
|
||||
"styles": "styles",
|
||||
"mbtiles": ""
|
||||
"mbtiles": "data",
|
||||
"pmtiles": "data"
|
||||
},
|
||||
"domains": [
|
||||
"localhost:8080",
|
||||
|
|
@ -180,11 +181,27 @@ Each item in this object defines one style (map). It can have the following opti
|
|||
``data``
|
||||
========
|
||||
|
||||
Each item specifies one data source which should be made accessible by the server. It has the following options:
|
||||
Each item specifies one data source which should be made accessible by the server. It has to have one of the following options:
|
||||
|
||||
* ``mbtiles`` -- name of the mbtiles file [required]
|
||||
* ``mbtiles`` -- name of the mbtiles file
|
||||
* ``pmtiles`` -- name of the pmtiles file or url.
|
||||
|
||||
The mbtiles file does not need to be specified here unless you explicitly want to serve the raw data.
|
||||
For example::
|
||||
|
||||
"data": {
|
||||
"source1": {
|
||||
"mbtiles": "source1.mbtiles"
|
||||
},
|
||||
"source2": {
|
||||
"pmtiles": "source2.pmtiles"
|
||||
},
|
||||
"source3": {
|
||||
"pmtiles": "https://foo.lan/source3.pmtiles"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
The data source does not need to be specified here unless you explicitly want to serve the raw data.
|
||||
|
||||
Referencing local files from style JSON
|
||||
=======================================
|
||||
|
|
@ -194,21 +211,46 @@ You can link various data sources from the style JSON (for example even remote T
|
|||
MBTiles
|
||||
-------
|
||||
|
||||
To specify that you want to use local mbtiles, use to following syntax: ``mbtiles://switzerland.mbtiles``.
|
||||
The TileServer-GL will try to find the file ``switzerland.mbtiles`` in ``root`` + ``mbtiles`` path.
|
||||
To specify that you want to use local mbtiles, use to following syntax: ``mbtiles://source1.mbtiles``.
|
||||
TileServer-GL will try to find the file ``source1.mbtiles`` in ``root`` + ``mbtiles`` path.
|
||||
|
||||
For example::
|
||||
|
||||
"sources": {
|
||||
"source1": {
|
||||
"url": "mbtiles://switzerland.mbtiles",
|
||||
"url": "mbtiles://source1.mbtiles",
|
||||
"type": "vector"
|
||||
}
|
||||
}
|
||||
|
||||
Alternatively, you can use ``mbtiles://{zurich-vector}`` to reference existing data object from the config.
|
||||
In this case, the server will look into the ``config.json`` to determine what mbtiles file to use.
|
||||
For the config above, this is equivalent to ``mbtiles://zurich.mbtiles``.
|
||||
Alternatively, you can use ``mbtiles://{source1}`` to reference existing data object from the config.
|
||||
In this case, the server will look into the ``config.json`` to determine what file to use by data id.
|
||||
For the config above, this is equivalent to ``mbtiles://source1.mbtiles``.
|
||||
|
||||
PMTiles
|
||||
-------
|
||||
|
||||
To specify that you want to use local pmtiles, use to following syntax: ``pmtiles://source2.pmtiles``.
|
||||
TileServer-GL will try to find the file ``source2.pmtiles`` in ``root`` + ``pmtiles`` path.
|
||||
|
||||
To specify that you want to use a url based pmtiles, use to following syntax: ``pmtiles://https://foo.lan/source3.pmtiles``.
|
||||
|
||||
For example::
|
||||
|
||||
"sources": {
|
||||
"source2": {
|
||||
"url": "pmtiles://source2.pmtiles",
|
||||
"type": "vector"
|
||||
},
|
||||
"source3": {
|
||||
"url": "pmtiles://https://foo.lan/source3.pmtiles",
|
||||
"type": "vector"
|
||||
},
|
||||
}
|
||||
|
||||
Alternatively, you can use ``pmtiles://{source2}`` to reference existing data object from the config.
|
||||
In this case, the server will look into the ``config.json`` to determine what file to use by data id.
|
||||
For the config above, this is equivalent to ``pmtiles://source2.mbtiles``.
|
||||
|
||||
Sprites
|
||||
-------
|
||||
|
|
|
|||
54
package-lock.json
generated
54
package-lock.json
generated
|
|
@ -16,6 +16,7 @@
|
|||
"@mapbox/vector-tile": "1.3.1",
|
||||
"@maplibre/maplibre-gl-native": "5.2.0",
|
||||
"@maplibre/maplibre-gl-style-spec": "18.0.0",
|
||||
"@sindresorhus/fnv1a": "3.0.0",
|
||||
"advanced-pool": "0.3.3",
|
||||
"canvas": "2.11.2",
|
||||
"chokidar": "3.5.3",
|
||||
|
|
@ -28,6 +29,7 @@
|
|||
"http-shutdown": "1.2.2",
|
||||
"morgan": "1.10.0",
|
||||
"pbf": "3.2.1",
|
||||
"pmtiles": "2.11.0",
|
||||
"proj4": "2.9.1",
|
||||
"request": "2.88.2",
|
||||
"sanitize-filename": "1.6.3",
|
||||
|
|
@ -985,6 +987,17 @@
|
|||
"@octokit/openapi-types": "^12.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@sindresorhus/fnv1a": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/fnv1a/-/fnv1a-3.0.0.tgz",
|
||||
"integrity": "sha512-M6pmbdZqAryzjZ4ELAzrdCMoMZk5lH/fshKrapfSeXdf2W+GDqZvPmfXaNTZp43//FVbSwkTPwpEMnehSyskkQ==",
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/@tootallnate/once": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
|
||||
|
|
@ -3251,9 +3264,9 @@
|
|||
"dev": true
|
||||
},
|
||||
"node_modules/fast-fifo": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.0.tgz",
|
||||
"integrity": "sha512-IgfweLvEpwyA4WgiQe9Nx6VV2QkML2NkvZnk1oKnIzXgXdWxuhF7zw4DvLTPZJn6PIUneiAXPF24QmoEqHTjyw=="
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
|
||||
"integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="
|
||||
},
|
||||
"node_modules/fast-glob": {
|
||||
"version": "3.2.12",
|
||||
|
|
@ -3297,6 +3310,11 @@
|
|||
"reusify": "^1.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/fflate": {
|
||||
"version": "0.8.1",
|
||||
"resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.1.tgz",
|
||||
"integrity": "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="
|
||||
},
|
||||
"node_modules/file-entry-cache": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
|
||||
|
|
@ -5964,9 +5982,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/node-addon-api": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
|
||||
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
|
||||
"integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.6.7",
|
||||
|
|
@ -6630,6 +6648,14 @@
|
|||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/pmtiles": {
|
||||
"version": "2.11.0",
|
||||
"resolved": "https://registry.npmjs.org/pmtiles/-/pmtiles-2.11.0.tgz",
|
||||
"integrity": "sha512-dU9SzzaqmCGpdEuTnIba6bDHT6j09ZJFIXxwGpvkiEnce3ZnBB1VKt6+EOmJGueriweaZLAMTUmKVElU2CBe0g==",
|
||||
"dependencies": {
|
||||
"fflate": "^0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/prebuild-install": {
|
||||
"version": "7.1.1",
|
||||
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
|
||||
|
|
@ -7492,6 +7518,11 @@
|
|||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/sharp/node_modules/node-addon-api": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
|
||||
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
|
||||
},
|
||||
"node_modules/sharp/node_modules/simple-get": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
|
||||
|
|
@ -7862,11 +7893,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/sqlite3/node_modules/node-addon-api": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
|
||||
"integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="
|
||||
},
|
||||
"node_modules/sshpk": {
|
||||
"version": "1.17.0",
|
||||
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz",
|
||||
|
|
@ -7912,9 +7938,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/streamx": {
|
||||
"version": "2.15.0",
|
||||
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.0.tgz",
|
||||
"integrity": "sha512-HcxY6ncGjjklGs1xsP1aR71INYcsXFJet5CU1CHqihQ2J5nOsbd4OjgjHO42w/4QNv9gZb3BueV+Vxok5pLEXg==",
|
||||
"version": "2.15.1",
|
||||
"resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.1.tgz",
|
||||
"integrity": "sha512-fQMzy2O/Q47rgwErk/eGeLu/roaFWV0jVsogDmrszM9uIw8L5OA+t+V93MgYlufNptfjmYR1tOMWhei/Eh7TQA==",
|
||||
"dependencies": {
|
||||
"fast-fifo": "^1.1.0",
|
||||
"queue-tick": "^1.0.1"
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
"@mapbox/vector-tile": "1.3.1",
|
||||
"@maplibre/maplibre-gl-native": "5.2.0",
|
||||
"@maplibre/maplibre-gl-style-spec": "18.0.0",
|
||||
"@sindresorhus/fnv1a": "3.0.0",
|
||||
"advanced-pool": "0.3.3",
|
||||
"canvas": "2.11.2",
|
||||
"chokidar": "3.5.3",
|
||||
|
|
@ -37,6 +38,7 @@
|
|||
"http-shutdown": "1.2.2",
|
||||
"morgan": "1.10.0",
|
||||
"pbf": "3.2.1",
|
||||
"pmtiles": "2.11.0",
|
||||
"proj4": "2.9.1",
|
||||
"request": "2.88.2",
|
||||
"sanitize-filename": "1.6.3",
|
||||
|
|
|
|||
|
|
@ -77,8 +77,9 @@
|
|||
<img src="{{public_url}}images/placeholder.png{{&key_query}}" alt="{{name}} preview" />
|
||||
{{/if}}
|
||||
<div class="details">
|
||||
<h3>{{name}}</h3>
|
||||
<p class="identifier">identifier: {{@key}}{{#if formatted_filesize}} | size: {{formatted_filesize}}{{/if}} | type: {{#is_vector}}vector{{/is_vector}}{{^is_vector}}raster{{/is_vector}} data</p>
|
||||
<h3>{{tileJSON.name}}</h3>
|
||||
<div class="identifier">identifier: {{@key}}{{#if formatted_filesize}} | size: {{formatted_filesize}}{{/if}}</div>
|
||||
<div class="identifier">type: {{#is_vector}}vector{{/is_vector}}{{^is_vector}}raster{{/is_vector}} data {{#if source_type}} | ext: {{source_type}}{{/if}}</div>
|
||||
<p class="services">
|
||||
services: <a href="{{public_url}}data/{{@key}}.json{{&../key_query}}">TileJSON</a>
|
||||
{{#if wmts_link}}
|
||||
|
|
|
|||
175
src/main.js
175
src/main.js
|
|
@ -7,8 +7,9 @@ import path from 'path';
|
|||
import { fileURLToPath } from 'url';
|
||||
import request from 'request';
|
||||
import { server } from './server.js';
|
||||
|
||||
import MBTiles from '@mapbox/mbtiles';
|
||||
import { isValidHttpUrl } from './utils.js';
|
||||
import { PMtilesOpen, GetPMtilesInfo } from './pmtiles_adapter.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
|
@ -25,9 +26,15 @@ import { program } from 'commander';
|
|||
program
|
||||
.description('tileserver-gl startup options')
|
||||
.usage('tileserver-gl [mbtiles] [options]')
|
||||
.option(
|
||||
'--file <file>',
|
||||
'MBTiles or PMTiles file\n' +
|
||||
'\t ignored if the configuration file is also specified',
|
||||
)
|
||||
.option(
|
||||
'--mbtiles <file>',
|
||||
'MBTiles file (uses demo configuration);\n' +
|
||||
'(DEPRECIATED) MBTiles file\n' +
|
||||
'\t ignored if file is also specified' +
|
||||
'\t ignored if the configuration file is also specified',
|
||||
)
|
||||
.option(
|
||||
|
|
@ -55,7 +62,7 @@ const opts = program.opts();
|
|||
|
||||
console.log(`Starting ${packageJson.name} v${packageJson.version}`);
|
||||
|
||||
const startServer = (configPath, config) => {
|
||||
const StartServer = (configPath, config) => {
|
||||
let publicUrl = opts.public_url;
|
||||
if (publicUrl && publicUrl.lastIndexOf('/') !== publicUrl.length - 1) {
|
||||
publicUrl += '/';
|
||||
|
|
@ -74,37 +81,27 @@ const startServer = (configPath, config) => {
|
|||
});
|
||||
};
|
||||
|
||||
const startWithMBTiles = (mbtilesFile) => {
|
||||
console.log(`[INFO] Automatically creating config file for ${mbtilesFile}`);
|
||||
const StartWithInputFile = async (inputFile) => {
|
||||
console.log(`[INFO] Automatically creating config file for ${inputFile}`);
|
||||
console.log(`[INFO] Only a basic preview style will be used.`);
|
||||
console.log(
|
||||
`[INFO] See documentation to learn how to create config.json file.`,
|
||||
);
|
||||
|
||||
mbtilesFile = path.resolve(process.cwd(), mbtilesFile);
|
||||
let inputFilePath;
|
||||
if (isValidHttpUrl(inputFile)) {
|
||||
inputFilePath = process.cwd();
|
||||
} else {
|
||||
inputFile = path.resolve(process.cwd(), inputFile);
|
||||
inputFilePath = path.dirname(inputFile);
|
||||
|
||||
const mbtilesStats = fs.statSync(mbtilesFile);
|
||||
if (!mbtilesStats.isFile() || mbtilesStats.size === 0) {
|
||||
console.log(`ERROR: Not valid MBTiles file: ${mbtilesFile}`);
|
||||
const inputFileStats = fs.statSync(inputFile);
|
||||
if (!inputFileStats.isFile() || inputFileStats.size === 0) {
|
||||
console.log(`ERROR: Not a valid input file: `);
|
||||
process.exit(1);
|
||||
}
|
||||
const instance = new MBTiles(mbtilesFile + '?mode=ro', (err) => {
|
||||
if (err) {
|
||||
console.log('ERROR: Unable to open MBTiles.');
|
||||
console.log(`Make sure ${path.basename(mbtilesFile)} is valid MBTiles.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
instance.getInfo((err, info) => {
|
||||
if (err || !info) {
|
||||
console.log('ERROR: Metadata missing in the MBTiles.');
|
||||
console.log(
|
||||
`Make sure ${path.basename(mbtilesFile)} is valid MBTiles.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const bounds = info.bounds;
|
||||
|
||||
const styleDir = path.resolve(
|
||||
__dirname,
|
||||
'../node_modules/tileserver-gl-styles/',
|
||||
|
|
@ -116,19 +113,98 @@ const startWithMBTiles = (mbtilesFile) => {
|
|||
root: styleDir,
|
||||
fonts: 'fonts',
|
||||
styles: 'styles',
|
||||
mbtiles: path.dirname(mbtilesFile),
|
||||
mbtiles: inputFilePath,
|
||||
pmtiles: inputFilePath,
|
||||
},
|
||||
},
|
||||
styles: {},
|
||||
data: {},
|
||||
};
|
||||
|
||||
const extension = inputFile.split('.').pop().toLowerCase();
|
||||
if (extension === 'pmtiles') {
|
||||
let FileOpenInfo = PMtilesOpen(inputFile);
|
||||
const metadata = await GetPMtilesInfo(FileOpenInfo);
|
||||
|
||||
if (
|
||||
metadata.format === 'pbf' &&
|
||||
metadata.name.toLowerCase().indexOf('openmaptiles') > -1
|
||||
) {
|
||||
if (isValidHttpUrl(inputFile)) {
|
||||
config['data'][`v3`] = {
|
||||
pmtiles: inputFile,
|
||||
};
|
||||
} else {
|
||||
config['data'][`v3`] = {
|
||||
pmtiles: path.basename(inputFile),
|
||||
};
|
||||
}
|
||||
|
||||
const styles = fs.readdirSync(path.resolve(styleDir, 'styles'));
|
||||
for (const styleName of styles) {
|
||||
const styleFileRel = styleName + '/style.json';
|
||||
const styleFile = path.resolve(styleDir, 'styles', styleFileRel);
|
||||
if (fs.existsSync(styleFile)) {
|
||||
config['styles'][styleName] = {
|
||||
style: styleFileRel,
|
||||
tilejson: {
|
||||
bounds: metadata.bounds,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`WARN: PMTiles not in "openmaptiles" format. Serving raw data only...`,
|
||||
);
|
||||
if (isValidHttpUrl(inputFile)) {
|
||||
config['data'][(metadata.id || 'pmtiles').replace(/[?/:]/g, '_')] = {
|
||||
pmtiles: inputFile,
|
||||
};
|
||||
} else {
|
||||
config['data'][(metadata.id || 'pmtiles').replace(/[?/:]/g, '_')] = {
|
||||
pmtiles: path.basename(inputFile),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.verbose) {
|
||||
console.log(JSON.stringify(config, undefined, 2));
|
||||
} else {
|
||||
console.log('Run with --verbose to see the config file here.');
|
||||
}
|
||||
|
||||
return StartServer(null, config);
|
||||
} else {
|
||||
if (isValidHttpUrl(inputFile)) {
|
||||
console.log(
|
||||
`ERROR: MBTiles does not support web based files. "${inputFile}" is not a valid data file.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const instance = new MBTiles(inputFile + '?mode=ro', (err) => {
|
||||
if (err) {
|
||||
console.log('ERROR: Unable to open MBTiles.');
|
||||
console.log(`Make sure ${path.basename(inputFile)} is valid MBTiles.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
instance.getInfo((err, info) => {
|
||||
if (err || !info) {
|
||||
console.log('ERROR: Metadata missing in the MBTiles.');
|
||||
console.log(
|
||||
`Make sure ${path.basename(inputFile)} is valid MBTiles.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const bounds = info.bounds;
|
||||
|
||||
if (
|
||||
info.format === 'pbf' &&
|
||||
info.name.toLowerCase().indexOf('openmaptiles') > -1
|
||||
) {
|
||||
config['data'][`v3`] = {
|
||||
mbtiles: path.basename(mbtilesFile),
|
||||
mbtiles: path.basename(inputFile),
|
||||
};
|
||||
|
||||
const styles = fs.readdirSync(path.resolve(styleDir, 'styles'));
|
||||
|
|
@ -148,13 +224,8 @@ const startWithMBTiles = (mbtilesFile) => {
|
|||
console.log(
|
||||
`WARN: MBTiles not in "openmaptiles" format. Serving raw data only...`,
|
||||
);
|
||||
config['data'][
|
||||
(info.id || 'mbtiles')
|
||||
.replace(/\//g, '_')
|
||||
.replace(/:/g, '_')
|
||||
.replace(/\?/g, '_')
|
||||
] = {
|
||||
mbtiles: path.basename(mbtilesFile),
|
||||
config['data'][(info.id || 'mbtiles').replace(/[?/:]/g, '_')] = {
|
||||
mbtiles: path.basename(inputFile),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -164,45 +235,51 @@ const startWithMBTiles = (mbtilesFile) => {
|
|||
console.log('Run with --verbose to see the config file here.');
|
||||
}
|
||||
|
||||
return startServer(null, config);
|
||||
return StartServer(null, config);
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
fs.stat(path.resolve(opts.config), (err, stats) => {
|
||||
if (err || !stats.isFile() || stats.size === 0) {
|
||||
let mbtiles = opts.mbtiles;
|
||||
if (!mbtiles) {
|
||||
let inputFile;
|
||||
if (opts.file) {
|
||||
inputFile = opts.file;
|
||||
} else if (opts.mbtiles) {
|
||||
inputFile = opts.mbtiles;
|
||||
}
|
||||
|
||||
if (inputFile) {
|
||||
return StartWithInputFile(inputFile);
|
||||
} else {
|
||||
// try to find in the cwd
|
||||
const files = fs.readdirSync(process.cwd());
|
||||
for (const filename of files) {
|
||||
if (filename.endsWith('.mbtiles')) {
|
||||
const mbTilesStats = fs.statSync(filename);
|
||||
if (mbTilesStats.isFile() && mbTilesStats.size > 0) {
|
||||
mbtiles = filename;
|
||||
if (filename.endsWith('.mbtiles') || filename.endsWith('.pmtiles')) {
|
||||
const inputFilesStats = fs.statSync(filename);
|
||||
if (inputFilesStats.isFile() && inputFilesStats.size > 0) {
|
||||
inputFile = filename;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (mbtiles) {
|
||||
console.log(`No MBTiles specified, using ${mbtiles}`);
|
||||
return startWithMBTiles(mbtiles);
|
||||
if (inputFile) {
|
||||
console.log(`No input file specified, using ${inputFile}`);
|
||||
return StartWithInputFile(inputFile);
|
||||
} else {
|
||||
const url =
|
||||
'https://github.com/maptiler/tileserver-gl/releases/download/v1.3.0/zurich_switzerland.mbtiles';
|
||||
const filename = 'zurich_switzerland.mbtiles';
|
||||
const stream = fs.createWriteStream(filename);
|
||||
console.log(`No MBTiles found`);
|
||||
console.log(`No input file found`);
|
||||
console.log(`[DEMO] Downloading sample data (${filename}) from ${url}`);
|
||||
stream.on('finish', () => startWithMBTiles(filename));
|
||||
stream.on('finish', () => StartWithInputFile(filename));
|
||||
return request.get(url).pipe(stream);
|
||||
}
|
||||
}
|
||||
if (mbtiles) {
|
||||
return startWithMBTiles(mbtiles);
|
||||
}
|
||||
} else {
|
||||
console.log(`Using specified config file from ${opts.config}`);
|
||||
return startServer(opts.config, null);
|
||||
return StartServer(opts.config, null);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
151
src/pmtiles_adapter.js
Normal file
151
src/pmtiles_adapter.js
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
import fs from 'node:fs';
|
||||
import PMTiles from 'pmtiles';
|
||||
import { isValidHttpUrl } from './utils.js';
|
||||
|
||||
class PMTilesFileSource {
|
||||
constructor(fd) {
|
||||
this.fd = fd;
|
||||
}
|
||||
getKey() {
|
||||
return this.fd;
|
||||
}
|
||||
async getBytes(offset, length) {
|
||||
const buffer = Buffer.alloc(length);
|
||||
await ReadFileBytes(this.fd, buffer, offset);
|
||||
const ab = buffer.buffer.slice(
|
||||
buffer.byteOffset,
|
||||
buffer.byteOffset + buffer.byteLength,
|
||||
);
|
||||
return { data: ab };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param fd
|
||||
* @param buffer
|
||||
* @param offset
|
||||
*/
|
||||
async function ReadFileBytes(fd, buffer, offset) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.read(fd, buffer, 0, buffer.length, offset, (err) => {
|
||||
if (err) {
|
||||
return reject(err);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param FilePath
|
||||
*/
|
||||
export function PMtilesOpen(FilePath) {
|
||||
let pmtiles = undefined;
|
||||
|
||||
if (isValidHttpUrl(FilePath)) {
|
||||
const source = new PMTiles.FetchSource(FilePath);
|
||||
pmtiles = new PMTiles.PMTiles(source);
|
||||
} else {
|
||||
const fd = fs.openSync(FilePath, 'r');
|
||||
const source = new PMTilesFileSource(fd);
|
||||
pmtiles = new PMTiles.PMTiles(source);
|
||||
}
|
||||
return pmtiles;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pmtiles
|
||||
*/
|
||||
export async function GetPMtilesInfo(pmtiles) {
|
||||
const header = await pmtiles.getHeader();
|
||||
const metadata = await pmtiles.getMetadata();
|
||||
|
||||
//Add missing metadata from header
|
||||
metadata['format'] = GetPmtilesTileType(header.tileType).type;
|
||||
metadata['minzoom'] = header.minZoom;
|
||||
metadata['maxzoom'] = header.maxZoom;
|
||||
|
||||
if (header.minLon && header.minLat && header.maxLon && header.maxLat) {
|
||||
metadata['bounds'] = [
|
||||
header.minLon,
|
||||
header.minLat,
|
||||
header.maxLon,
|
||||
header.maxLat,
|
||||
];
|
||||
} else {
|
||||
metadata['bounds'] = [-180, -85.05112877980659, 180, 85.0511287798066];
|
||||
}
|
||||
|
||||
if (header.centerZoom) {
|
||||
metadata['center'] = [
|
||||
header.centerLon,
|
||||
header.centerLat,
|
||||
header.centerZoom,
|
||||
];
|
||||
} else {
|
||||
metadata['center'] = [
|
||||
header.centerLon,
|
||||
header.centerLat,
|
||||
parseInt(metadata['maxzoom']) / 2,
|
||||
];
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param pmtiles
|
||||
* @param z
|
||||
* @param x
|
||||
* @param y
|
||||
*/
|
||||
export async function GetPMtilesTile(pmtiles, z, x, y) {
|
||||
const header = await pmtiles.getHeader();
|
||||
const TileType = GetPmtilesTileType(header.tileType);
|
||||
let zxyTile = await pmtiles.getZxy(z, x, y);
|
||||
if (zxyTile && zxyTile.data) {
|
||||
zxyTile = Buffer.from(zxyTile.data);
|
||||
} else {
|
||||
zxyTile = undefined;
|
||||
}
|
||||
return { data: zxyTile, header: TileType.header };
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param typenum
|
||||
*/
|
||||
function GetPmtilesTileType(typenum) {
|
||||
let head = {};
|
||||
let tileType;
|
||||
switch (typenum) {
|
||||
case 0:
|
||||
tileType = 'Unknown';
|
||||
break;
|
||||
case 1:
|
||||
tileType = 'pbf';
|
||||
head['Content-Type'] = 'application/x-protobuf';
|
||||
break;
|
||||
case 2:
|
||||
tileType = 'png';
|
||||
head['Content-Type'] = 'image/png';
|
||||
break;
|
||||
case 3:
|
||||
tileType = 'jpeg';
|
||||
head['Content-Type'] = 'image/jpeg';
|
||||
break;
|
||||
case 4:
|
||||
tileType = 'webp';
|
||||
head['Content-Type'] = 'image/webp';
|
||||
break;
|
||||
case 5:
|
||||
tileType = 'avif';
|
||||
head['Content-Type'] = 'image/avif';
|
||||
break;
|
||||
}
|
||||
return { type: tileType, header: head };
|
||||
}
|
||||
|
|
@ -10,7 +10,12 @@ import MBTiles from '@mapbox/mbtiles';
|
|||
import Pbf from 'pbf';
|
||||
import { VectorTile } from '@mapbox/vector-tile';
|
||||
|
||||
import { getTileUrls, fixTileJSONCenter } from './utils.js';
|
||||
import { getTileUrls, isValidHttpUrl, fixTileJSONCenter } from './utils.js';
|
||||
import {
|
||||
PMtilesOpen,
|
||||
GetPMtilesInfo,
|
||||
GetPMtilesTile,
|
||||
} from './pmtiles_adapter.js';
|
||||
|
||||
export const serve_data = {
|
||||
init: (options, repo) => {
|
||||
|
|
@ -18,7 +23,7 @@ export const serve_data = {
|
|||
|
||||
app.get(
|
||||
'/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)',
|
||||
(req, res, next) => {
|
||||
async (req, res, next) => {
|
||||
const item = repo[req.params.id];
|
||||
if (!item) {
|
||||
return res.sendStatus(404);
|
||||
|
|
@ -48,6 +53,53 @@ export const serve_data = {
|
|||
) {
|
||||
return res.status(404).send('Out of bounds');
|
||||
}
|
||||
if (item.source_type === 'pmtiles') {
|
||||
let tileinfo = await GetPMtilesTile(item.source, z, x, y);
|
||||
if (tileinfo == undefined || tileinfo.data == undefined) {
|
||||
return res.status(404).send('Not found');
|
||||
} else {
|
||||
let data = tileinfo.data;
|
||||
let headers = tileinfo.header;
|
||||
if (tileJSONFormat === 'pbf') {
|
||||
if (options.dataDecoratorFunc) {
|
||||
data = options.dataDecoratorFunc(id, 'data', data, z, x, y);
|
||||
}
|
||||
}
|
||||
if (format === 'pbf') {
|
||||
headers['Content-Type'] = 'application/x-protobuf';
|
||||
} else if (format === 'geojson') {
|
||||
headers['Content-Type'] = 'application/json';
|
||||
|
||||
if (isGzipped) {
|
||||
data = zlib.unzipSync(data);
|
||||
isGzipped = false;
|
||||
}
|
||||
|
||||
const tile = new VectorTile(new Pbf(data));
|
||||
const geojson = {
|
||||
type: 'FeatureCollection',
|
||||
features: [],
|
||||
};
|
||||
for (const layerName in tile.layers) {
|
||||
const layer = tile.layers[layerName];
|
||||
for (let i = 0; i < layer.length; i++) {
|
||||
const feature = layer.feature(i);
|
||||
const featureGeoJSON = feature.toGeoJSON(x, y, z);
|
||||
featureGeoJSON.properties.layer = layerName;
|
||||
geojson.features.push(featureGeoJSON);
|
||||
}
|
||||
}
|
||||
data = JSON.stringify(geojson);
|
||||
}
|
||||
delete headers['ETag']; // do not trust the tile ETag -- regenerate
|
||||
headers['Content-Encoding'] = 'gzip';
|
||||
res.set(headers);
|
||||
|
||||
data = zlib.gzipSync(data);
|
||||
|
||||
return res.status(200).send(data);
|
||||
}
|
||||
} else if (item.source_type === 'mbtiles') {
|
||||
item.source.getTile(z, x, y, (err, data, headers) => {
|
||||
let isGzipped;
|
||||
if (err) {
|
||||
|
|
@ -106,13 +158,13 @@ export const serve_data = {
|
|||
|
||||
if (!isGzipped) {
|
||||
data = zlib.gzipSync(data);
|
||||
isGzipped = true;
|
||||
}
|
||||
|
||||
return res.status(200).send(data);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -137,19 +189,65 @@ export const serve_data = {
|
|||
|
||||
return app;
|
||||
},
|
||||
add: (options, repo, params, id, publicUrl) => {
|
||||
const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles);
|
||||
add: async (options, repo, params, id, publicUrl) => {
|
||||
let inputFile;
|
||||
let inputType;
|
||||
if (params.pmtiles) {
|
||||
inputType = 'pmtiles';
|
||||
if (isValidHttpUrl(params.pmtiles)) {
|
||||
inputFile = params.pmtiles;
|
||||
} else {
|
||||
inputFile = path.resolve(options.paths.pmtiles, params.pmtiles);
|
||||
}
|
||||
} else if (params.mbtiles) {
|
||||
inputType = 'mbtiles';
|
||||
if (isValidHttpUrl(params.mbtiles)) {
|
||||
console.log(
|
||||
`ERROR: MBTiles does not support web based files. "${params.mbtiles}" is not a valid data file.`,
|
||||
);
|
||||
process.exit(1);
|
||||
} else {
|
||||
inputFile = path.resolve(options.paths.mbtiles, params.mbtiles);
|
||||
}
|
||||
}
|
||||
|
||||
let tileJSON = {
|
||||
tiles: params.domains || options.domains,
|
||||
};
|
||||
|
||||
const mbtilesFileStats = fs.statSync(mbtilesFile);
|
||||
if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
|
||||
throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
|
||||
if (!isValidHttpUrl(inputFile)) {
|
||||
const inputFileStats = fs.statSync(inputFile);
|
||||
if (!inputFileStats.isFile() || inputFileStats.size === 0) {
|
||||
throw Error(`Not valid input file: "${inputFile}"`);
|
||||
}
|
||||
}
|
||||
|
||||
let source;
|
||||
let source_type;
|
||||
if (inputType === 'pmtiles') {
|
||||
source = PMtilesOpen(inputFile);
|
||||
source_type = 'pmtiles';
|
||||
const metadata = await GetPMtilesInfo(source);
|
||||
|
||||
tileJSON['name'] = id;
|
||||
tileJSON['format'] = 'pbf';
|
||||
Object.assign(tileJSON, metadata);
|
||||
|
||||
tileJSON['tilejson'] = '2.0.0';
|
||||
delete tileJSON['filesize'];
|
||||
delete tileJSON['mtime'];
|
||||
delete tileJSON['scheme'];
|
||||
|
||||
Object.assign(tileJSON, params.tilejson || {});
|
||||
fixTileJSONCenter(tileJSON);
|
||||
|
||||
if (options.dataDecoratorFunc) {
|
||||
tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
|
||||
}
|
||||
} else if (inputType === 'mbtiles') {
|
||||
source_type = 'mbtiles';
|
||||
const sourceInfoPromise = new Promise((resolve, reject) => {
|
||||
source = new MBTiles(mbtilesFile + '?mode=ro', (err) => {
|
||||
source = new MBTiles(inputFile + '?mode=ro', (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
return;
|
||||
|
|
@ -180,12 +278,14 @@ export const serve_data = {
|
|||
});
|
||||
});
|
||||
|
||||
return sourceInfoPromise.then(() => {
|
||||
await sourceInfoPromise;
|
||||
}
|
||||
|
||||
repo[id] = {
|
||||
tileJSON,
|
||||
publicUrl,
|
||||
source,
|
||||
source_type,
|
||||
};
|
||||
});
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import path from 'path';
|
|||
import url from 'url';
|
||||
import util from 'util';
|
||||
import zlib from 'zlib';
|
||||
import sharp from 'sharp'; // sharp has to be required before node-canvas. see https://github.com/lovell/sharp/issues/371
|
||||
import sharp from 'sharp'; // sharp has to be required before node-canvas on linux but after it on windows. see https://github.com/lovell/sharp/issues/371
|
||||
import { createCanvas, Image } from 'canvas';
|
||||
import clone from 'clone';
|
||||
import Color from 'color';
|
||||
|
|
@ -18,7 +18,17 @@ import MBTiles from '@mapbox/mbtiles';
|
|||
import polyline from '@mapbox/polyline';
|
||||
import proj4 from 'proj4';
|
||||
import request from 'request';
|
||||
import { getFontsPbf, getTileUrls, fixTileJSONCenter } from './utils.js';
|
||||
import {
|
||||
getFontsPbf,
|
||||
getTileUrls,
|
||||
isValidHttpUrl,
|
||||
fixTileJSONCenter,
|
||||
} from './utils.js';
|
||||
import {
|
||||
PMtilesOpen,
|
||||
GetPMtilesInfo,
|
||||
GetPMtilesTile,
|
||||
} from './pmtiles_adapter.js';
|
||||
|
||||
const FLOAT_PATTERN = '[+-]?(?:\\d+|\\d+.?\\d+)';
|
||||
const PATH_PATTERN =
|
||||
|
|
@ -1207,11 +1217,12 @@ export const serve_rendered = {
|
|||
|
||||
return Promise.all([fontListingPromise]).then(() => app);
|
||||
},
|
||||
add: (options, repo, params, id, publicUrl, dataResolver) => {
|
||||
add: async (options, repo, params, id, publicUrl, dataResolver) => {
|
||||
const map = {
|
||||
renderers: [],
|
||||
renderers_static: [],
|
||||
sources: {},
|
||||
source_types: {},
|
||||
};
|
||||
|
||||
let styleJSON;
|
||||
|
|
@ -1220,7 +1231,7 @@ export const serve_rendered = {
|
|||
const renderer = new mlgl.Map({
|
||||
mode: mode,
|
||||
ratio: ratio,
|
||||
request: (req, callback) => {
|
||||
request: async (req, callback) => {
|
||||
const protocol = req.url.split(':')[0];
|
||||
// console.log('Handling request:', req);
|
||||
if (protocol === 'sprites') {
|
||||
|
|
@ -1247,15 +1258,54 @@ export const serve_rendered = {
|
|||
callback(err, { data: null });
|
||||
},
|
||||
);
|
||||
} else if (protocol === 'mbtiles') {
|
||||
} else if (protocol === 'mbtiles' || protocol === 'pmtiles') {
|
||||
const parts = req.url.split('/');
|
||||
const sourceId = parts[2];
|
||||
const source = map.sources[sourceId];
|
||||
const source_type = map.source_types[sourceId];
|
||||
const sourceInfo = styleJSON.sources[sourceId];
|
||||
|
||||
const z = parts[3] | 0;
|
||||
const x = parts[4] | 0;
|
||||
const y = parts[5].split('.')[0] | 0;
|
||||
const format = parts[5].split('.')[1];
|
||||
|
||||
if (source_type === 'pmtiles') {
|
||||
let tileinfo = await GetPMtilesTile(source, z, x, y);
|
||||
let data = tileinfo.data;
|
||||
let headers = tileinfo.header;
|
||||
if (data == undefined) {
|
||||
if (options.verbose)
|
||||
console.log('MBTiles error, serving empty', err);
|
||||
createEmptyResponse(
|
||||
sourceInfo.format,
|
||||
sourceInfo.color,
|
||||
callback,
|
||||
);
|
||||
return;
|
||||
} else {
|
||||
const response = {};
|
||||
response.data = data;
|
||||
if (headers['Last-Modified']) {
|
||||
response.modified = new Date(headers['Last-Modified']);
|
||||
}
|
||||
|
||||
if (format === 'pbf') {
|
||||
if (options.dataDecoratorFunc) {
|
||||
response.data = options.dataDecoratorFunc(
|
||||
sourceId,
|
||||
'data',
|
||||
response.data,
|
||||
z,
|
||||
x,
|
||||
y,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
callback(null, response);
|
||||
}
|
||||
} else if (source_type === 'mbtiles') {
|
||||
source.getTile(z, x, y, (err, data, headers) => {
|
||||
if (err) {
|
||||
if (options.verbose)
|
||||
|
|
@ -1301,6 +1351,7 @@ export const serve_rendered = {
|
|||
|
||||
callback(null, response);
|
||||
});
|
||||
}
|
||||
} else if (protocol === 'http' || protocol === 'https') {
|
||||
request(
|
||||
{
|
||||
|
|
@ -1416,43 +1467,92 @@ export const serve_rendered = {
|
|||
|
||||
const queue = [];
|
||||
for (const name of Object.keys(styleJSON.sources)) {
|
||||
let source_type;
|
||||
let source = styleJSON.sources[name];
|
||||
const url = source.url;
|
||||
|
||||
if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
|
||||
// found mbtiles source, replace with info from local file
|
||||
let url = source.url;
|
||||
if (
|
||||
url &&
|
||||
(url.startsWith('pmtiles://') || url.startsWith('mbtiles://'))
|
||||
) {
|
||||
// found pmtiles or mbtiles source, replace with info from local file
|
||||
delete source.url;
|
||||
|
||||
let mbtilesFile = url.substring('mbtiles://'.length);
|
||||
const fromData =
|
||||
mbtilesFile[0] === '{' && mbtilesFile[mbtilesFile.length - 1] === '}';
|
||||
|
||||
if (fromData) {
|
||||
mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
|
||||
const mapsTo = (params.mapping || {})[mbtilesFile];
|
||||
if (mapsTo) {
|
||||
mbtilesFile = mapsTo;
|
||||
let dataId = url.replace('pmtiles://', '').replace('mbtiles://', '');
|
||||
if (dataId.startsWith('{') && dataId.endsWith('}')) {
|
||||
dataId = dataId.slice(1, -1);
|
||||
}
|
||||
mbtilesFile = dataResolver(mbtilesFile);
|
||||
if (!mbtilesFile) {
|
||||
console.error(`ERROR: data "${mbtilesFile}" not found!`);
|
||||
|
||||
const mapsTo = (params.mapping || {})[dataId];
|
||||
if (mapsTo) {
|
||||
dataId = mapsTo;
|
||||
}
|
||||
|
||||
let inputFile;
|
||||
const DataInfo = dataResolver(dataId);
|
||||
if (DataInfo.inputfile) {
|
||||
inputFile = DataInfo.inputfile;
|
||||
source_type = DataInfo.filetype;
|
||||
} else {
|
||||
console.error(`ERROR: data "${inputFile}" not found!`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!isValidHttpUrl(inputFile)) {
|
||||
const inputFileStats = fs.statSync(inputFile);
|
||||
if (!inputFileStats.isFile() || inputFileStats.size === 0) {
|
||||
throw Error(`Not valid PMTiles file: "${inputFile}"`);
|
||||
}
|
||||
}
|
||||
|
||||
if (source_type === 'pmtiles') {
|
||||
map.sources[name] = PMtilesOpen(inputFile);
|
||||
map.source_types[name] = 'pmtiles';
|
||||
const metadata = await GetPMtilesInfo(map.sources[name]);
|
||||
|
||||
if (!repoobj.dataProjWGStoInternalWGS && metadata.proj4) {
|
||||
// how to do this for multiple sources with different proj4 defs?
|
||||
const to3857 = proj4('EPSG:3857');
|
||||
const toDataProj = proj4(metadata.proj4);
|
||||
repoobj.dataProjWGStoInternalWGS = (xy) =>
|
||||
to3857.inverse(toDataProj.forward(xy));
|
||||
}
|
||||
|
||||
const type = source.type;
|
||||
Object.assign(source, metadata);
|
||||
source.type = type;
|
||||
source.tiles = [
|
||||
// meta url which will be detected when requested
|
||||
`pmtiles://${name}/{z}/{x}/{y}.${metadata.format || 'pbf'}`,
|
||||
];
|
||||
delete source.scheme;
|
||||
|
||||
if (
|
||||
!attributionOverride &&
|
||||
source.attribution &&
|
||||
source.attribution.length > 0
|
||||
) {
|
||||
if (!tileJSON.attribution.includes(source.attribution)) {
|
||||
if (tileJSON.attribution.length > 0) {
|
||||
tileJSON.attribution += ' | ';
|
||||
}
|
||||
tileJSON.attribution += source.attribution;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
queue.push(
|
||||
new Promise((resolve, reject) => {
|
||||
mbtilesFile = path.resolve(options.paths.mbtiles, mbtilesFile);
|
||||
const mbtilesFileStats = fs.statSync(mbtilesFile);
|
||||
if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
|
||||
throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
|
||||
inputFile = path.resolve(options.paths.mbtiles, inputFile);
|
||||
const inputFileStats = fs.statSync(inputFile);
|
||||
if (!inputFileStats.isFile() || inputFileStats.size === 0) {
|
||||
throw Error(`Not valid MBTiles file: "${inputFile}"`);
|
||||
}
|
||||
map.sources[name] = new MBTiles(mbtilesFile + '?mode=ro', (err) => {
|
||||
map.sources[name] = new MBTiles(inputFile + '?mode=ro', (err) => {
|
||||
map.sources[name].getInfo((err, info) => {
|
||||
if (err) {
|
||||
console.error(err);
|
||||
return;
|
||||
}
|
||||
map.source_types[name] = 'mbtiles';
|
||||
|
||||
if (!repoobj.dataProjWGStoInternalWGS && info.proj4) {
|
||||
// how to do this for multiple sources with different proj4 defs?
|
||||
|
|
@ -1472,7 +1572,11 @@ export const serve_rendered = {
|
|||
delete source.scheme;
|
||||
|
||||
if (options.dataDecoratorFunc) {
|
||||
source = options.dataDecoratorFunc(name, 'tilejson', source);
|
||||
source = options.dataDecoratorFunc(
|
||||
name,
|
||||
'tilejson',
|
||||
source,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
|
|
@ -1494,6 +1598,7 @@ export const serve_rendered = {
|
|||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const renderersReadyPromise = Promise.all(queue).then(() => {
|
||||
// standard and @2x tiles are much more usual -> default to larger pools
|
||||
|
|
|
|||
|
|
@ -110,20 +110,24 @@ export const serve_style = {
|
|||
|
||||
for (const name of Object.keys(styleJSON.sources)) {
|
||||
const source = styleJSON.sources[name];
|
||||
const url = source.url;
|
||||
if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
|
||||
let mbtilesFile = url.substring('mbtiles://'.length);
|
||||
const fromData =
|
||||
mbtilesFile[0] === '{' && mbtilesFile[mbtilesFile.length - 1] === '}';
|
||||
let url = source.url;
|
||||
if (
|
||||
url &&
|
||||
(url.startsWith('pmtiles://') || url.startsWith('mbtiles://'))
|
||||
) {
|
||||
const protocol = url.split(':')[0];
|
||||
|
||||
if (fromData) {
|
||||
mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
|
||||
const mapsTo = (params.mapping || {})[mbtilesFile];
|
||||
let dataId = url.replace('pmtiles://', '').replace('mbtiles://', '');
|
||||
if (dataId.startsWith('{') && dataId.endsWith('}')) {
|
||||
dataId = dataId.slice(1, -1);
|
||||
}
|
||||
|
||||
const mapsTo = (params.mapping || {})[dataId];
|
||||
if (mapsTo) {
|
||||
mbtilesFile = mapsTo;
|
||||
dataId = mapsTo;
|
||||
}
|
||||
}
|
||||
const identifier = reportTiles(mbtilesFile, fromData);
|
||||
|
||||
const identifier = reportTiles(dataId, protocol);
|
||||
if (!identifier) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
174
src/server.js
174
src/server.js
|
|
@ -6,7 +6,7 @@ process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, os.cpus().length * 1.5));
|
|||
|
||||
import fs from 'node:fs';
|
||||
import path from 'path';
|
||||
|
||||
import fnv1a from '@sindresorhus/fnv1a';
|
||||
import chokidar from 'chokidar';
|
||||
import clone from 'clone';
|
||||
import cors from 'cors';
|
||||
|
|
@ -19,7 +19,7 @@ import morgan from 'morgan';
|
|||
import { serve_data } from './serve_data.js';
|
||||
import { serve_style } from './serve_style.js';
|
||||
import { serve_font } from './serve_font.js';
|
||||
import { getTileUrls, getPublicUrl } from './utils.js';
|
||||
import { getTileUrls, getPublicUrl, isValidHttpUrl } from './utils.js';
|
||||
|
||||
import { fileURLToPath } from 'url';
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
|
|
@ -93,6 +93,7 @@ function start(opts) {
|
|||
paths.fonts = path.resolve(paths.root, paths.fonts || '');
|
||||
paths.sprites = path.resolve(paths.root, paths.sprites || '');
|
||||
paths.mbtiles = path.resolve(paths.root, paths.mbtiles || '');
|
||||
paths.pmtiles = path.resolve(paths.root, paths.pmtiles || '');
|
||||
paths.icons = path.resolve(paths.root, paths.icons || '');
|
||||
|
||||
const startupPromises = [];
|
||||
|
|
@ -109,6 +110,7 @@ function start(opts) {
|
|||
checkPath('fonts');
|
||||
checkPath('sprites');
|
||||
checkPath('mbtiles');
|
||||
checkPath('pmtiles');
|
||||
checkPath('icons');
|
||||
|
||||
/**
|
||||
|
|
@ -181,34 +183,43 @@ function start(opts) {
|
|||
item,
|
||||
id,
|
||||
opts.publicUrl,
|
||||
(mbtiles, fromData) => {
|
||||
(StyleSourceId, protocol) => {
|
||||
let dataItemId;
|
||||
for (const id of Object.keys(data)) {
|
||||
if (fromData) {
|
||||
if (id === mbtiles) {
|
||||
if (id === StyleSourceId) {
|
||||
// Style id was found in data ids, return that id
|
||||
dataItemId = id;
|
||||
}
|
||||
} else {
|
||||
if (data[id].mbtiles === mbtiles) {
|
||||
const fileType = Object.keys(data[id])[0];
|
||||
if (data[id][fileType] === StyleSourceId) {
|
||||
// Style id was found in data filename, return the id that filename belong to
|
||||
dataItemId = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (dataItemId) {
|
||||
// mbtiles exist in the data config
|
||||
// input files exists in the data config, return found id
|
||||
return dataItemId;
|
||||
} else {
|
||||
if (fromData || !allowMoreData) {
|
||||
if (!allowMoreData) {
|
||||
console.log(
|
||||
`ERROR: style "${item.style}" using unknown mbtiles "${mbtiles}"! Skipping...`,
|
||||
`ERROR: style "${item.style}" using unknown file "${StyleSourceId}"! Skipping...`,
|
||||
);
|
||||
return undefined;
|
||||
} else {
|
||||
let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles;
|
||||
while (data[id]) id += '_';
|
||||
let id =
|
||||
StyleSourceId.substr(0, StyleSourceId.lastIndexOf('.')) ||
|
||||
StyleSourceId;
|
||||
if (isValidHttpUrl(StyleSourceId)) {
|
||||
id =
|
||||
fnv1a(StyleSourceId) + '_' + id.replace(/^.*\/(.*)$/, '$1');
|
||||
}
|
||||
while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't
|
||||
//Add the new data source to the data array.
|
||||
data[id] = {
|
||||
mbtiles: mbtiles,
|
||||
[protocol]: StyleSourceId,
|
||||
};
|
||||
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
|
@ -229,14 +240,24 @@ function start(opts) {
|
|||
item,
|
||||
id,
|
||||
opts.publicUrl,
|
||||
(mbtiles) => {
|
||||
let mbtilesFile;
|
||||
(StyleSourceId) => {
|
||||
let fileType;
|
||||
let inputFile;
|
||||
for (const id of Object.keys(data)) {
|
||||
if (id === mbtiles) {
|
||||
mbtilesFile = data[id].mbtiles;
|
||||
fileType = Object.keys(data[id])[0];
|
||||
if (StyleSourceId == id) {
|
||||
inputFile = data[id][fileType];
|
||||
break;
|
||||
} else if (data[id][fileType] == StyleSourceId) {
|
||||
inputFile = data[id][fileType];
|
||||
break;
|
||||
}
|
||||
}
|
||||
return mbtilesFile;
|
||||
if (!isValidHttpUrl(inputFile)) {
|
||||
inputFile = path.resolve(options.paths[fileType], inputFile);
|
||||
}
|
||||
|
||||
return { inputfile: inputFile, filetype: fileType };
|
||||
},
|
||||
),
|
||||
);
|
||||
|
|
@ -264,8 +285,11 @@ function start(opts) {
|
|||
|
||||
for (const id of Object.keys(data)) {
|
||||
const item = data[id];
|
||||
if (!item.mbtiles || item.mbtiles.length === 0) {
|
||||
console.log(`Missing "mbtiles" property for ${id}`);
|
||||
const fileType = Object.keys(data[id])[0];
|
||||
if (!fileType || !(fileType === 'pmtiles' || fileType === 'mbtiles')) {
|
||||
console.log(
|
||||
`Missing "pmtiles" or "mbtiles" property for ${id} data source`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -424,14 +448,16 @@ function start(opts) {
|
|||
};
|
||||
|
||||
serveTemplate('/$', 'index', (req) => {
|
||||
const styles = clone(serving.styles || {});
|
||||
for (const id of Object.keys(styles)) {
|
||||
const style = styles[id];
|
||||
style.name = (serving.styles[id] || serving.rendered[id] || {}).name;
|
||||
style.serving_data = serving.styles[id];
|
||||
style.serving_rendered = serving.rendered[id];
|
||||
let styles = {};
|
||||
for (const id of Object.keys(serving.styles || {})) {
|
||||
let style = {
|
||||
...serving.styles[id],
|
||||
serving_data: serving.styles[id],
|
||||
serving_rendered: serving.rendered[id],
|
||||
};
|
||||
|
||||
if (style.serving_rendered) {
|
||||
const center = style.serving_rendered.tileJSON.center;
|
||||
const { center } = style.serving_rendered.tileJSON;
|
||||
if (center) {
|
||||
style.viewer_hash = `#${center[2]}/${center[1].toFixed(
|
||||
5,
|
||||
|
|
@ -451,40 +477,46 @@ function start(opts) {
|
|||
opts.publicUrl,
|
||||
)[0];
|
||||
}
|
||||
|
||||
styles[id] = style;
|
||||
}
|
||||
const data = clone(serving.data || {});
|
||||
for (const id of Object.keys(data)) {
|
||||
const data_ = data[id];
|
||||
const tilejson = data[id].tileJSON;
|
||||
const center = tilejson.center;
|
||||
|
||||
let datas = {};
|
||||
for (const id of Object.keys(serving.data || {})) {
|
||||
let data = Object.assign({}, serving.data[id]);
|
||||
|
||||
const { tileJSON } = serving.data[id];
|
||||
const { center } = tileJSON;
|
||||
|
||||
if (center) {
|
||||
data_.viewer_hash = `#${center[2]}/${center[1].toFixed(
|
||||
data.viewer_hash = `#${center[2]}/${center[1].toFixed(
|
||||
5,
|
||||
)}/${center[0].toFixed(5)}`;
|
||||
}
|
||||
data_.is_vector = tilejson.format === 'pbf';
|
||||
if (!data_.is_vector) {
|
||||
|
||||
data.is_vector = tileJSON.format === 'pbf';
|
||||
if (!data.is_vector) {
|
||||
if (center) {
|
||||
const centerPx = mercator.px([center[0], center[1]], center[2]);
|
||||
data_.thumbnail = `${center[2]}/${Math.floor(
|
||||
data.thumbnail = `${center[2]}/${Math.floor(
|
||||
centerPx[0] / 256,
|
||||
)}/${Math.floor(centerPx[1] / 256)}.${data_.tileJSON.format}`;
|
||||
)}/${Math.floor(centerPx[1] / 256)}.${tileJSON.format}`;
|
||||
}
|
||||
|
||||
data_.xyz_link = getTileUrls(
|
||||
data.xyz_link = getTileUrls(
|
||||
req,
|
||||
tilejson.tiles,
|
||||
tileJSON.tiles,
|
||||
`data/${id}`,
|
||||
tilejson.format,
|
||||
tileJSON.format,
|
||||
opts.publicUrl,
|
||||
{
|
||||
pbf: options.pbfAlias,
|
||||
},
|
||||
)[0];
|
||||
}
|
||||
if (data_.filesize) {
|
||||
if (data.filesize) {
|
||||
let suffix = 'kB';
|
||||
let size = parseInt(data_.filesize, 10) / 1024;
|
||||
let size = parseInt(tileJSON.filesize, 10) / 1024;
|
||||
if (size > 1024) {
|
||||
suffix = 'MB';
|
||||
size /= 1024;
|
||||
|
|
@ -493,26 +525,33 @@ function start(opts) {
|
|||
suffix = 'GB';
|
||||
size /= 1024;
|
||||
}
|
||||
data_.formatted_filesize = `${size.toFixed(2)} ${suffix}`;
|
||||
data.formatted_filesize = `${size.toFixed(2)} ${suffix}`;
|
||||
}
|
||||
|
||||
datas[id] = data;
|
||||
}
|
||||
|
||||
return {
|
||||
styles: Object.keys(styles).length ? styles : null,
|
||||
data: Object.keys(data).length ? data : null,
|
||||
data: Object.keys(datas).length ? datas : null,
|
||||
};
|
||||
});
|
||||
|
||||
serveTemplate('/styles/:id/$', 'viewer', (req) => {
|
||||
const id = req.params.id;
|
||||
const { id } = req.params;
|
||||
const style = clone(((serving.styles || {})[id] || {}).styleJSON);
|
||||
|
||||
if (!style) {
|
||||
return null;
|
||||
}
|
||||
style.id = id;
|
||||
style.name = (serving.styles[id] || serving.rendered[id]).name;
|
||||
style.serving_data = serving.styles[id];
|
||||
style.serving_rendered = serving.rendered[id];
|
||||
return style;
|
||||
|
||||
return {
|
||||
id,
|
||||
name: (serving.styles[id] || serving.rendered[id]).name,
|
||||
serving_data: serving.styles[id],
|
||||
serving_rendered: serving.rendered[id],
|
||||
...style,
|
||||
};
|
||||
});
|
||||
|
||||
/*
|
||||
|
|
@ -521,37 +560,49 @@ function start(opts) {
|
|||
});
|
||||
*/
|
||||
serveTemplate('/styles/:id/wmts.xml', 'wmts', (req) => {
|
||||
const id = req.params.id;
|
||||
const { id } = req.params;
|
||||
const wmts = clone((serving.styles || {})[id]);
|
||||
|
||||
if (!wmts) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (wmts.hasOwnProperty('serve_rendered') && !wmts.serve_rendered) {
|
||||
return null;
|
||||
}
|
||||
wmts.id = id;
|
||||
wmts.name = (serving.styles[id] || serving.rendered[id]).name;
|
||||
|
||||
let baseUrl;
|
||||
if (opts.publicUrl) {
|
||||
wmts.baseUrl = opts.publicUrl;
|
||||
baseUrl = opts.publicUrl;
|
||||
} else {
|
||||
wmts.baseUrl = `${
|
||||
baseUrl = `${
|
||||
req.get('X-Forwarded-Protocol')
|
||||
? req.get('X-Forwarded-Protocol')
|
||||
: req.protocol
|
||||
}://${req.get('host')}/`;
|
||||
}
|
||||
return wmts;
|
||||
|
||||
return {
|
||||
id,
|
||||
name: (serving.styles[id] || serving.rendered[id]).name,
|
||||
baseUrl,
|
||||
...wmts,
|
||||
};
|
||||
});
|
||||
|
||||
serveTemplate('/data/:id/$', 'data', (req) => {
|
||||
const id = req.params.id;
|
||||
const data = clone(serving.data[id]);
|
||||
const { id } = req.params;
|
||||
const data = serving.data[id];
|
||||
|
||||
if (!data) {
|
||||
return null;
|
||||
}
|
||||
data.id = id;
|
||||
data.is_vector = data.tileJSON.format === 'pbf';
|
||||
return data;
|
||||
|
||||
return {
|
||||
id,
|
||||
is_vector: data.tileJSON.format === 'pbf',
|
||||
...data,
|
||||
};
|
||||
});
|
||||
|
||||
let startupComplete = false;
|
||||
|
|
@ -559,6 +610,7 @@ function start(opts) {
|
|||
console.log('Startup complete');
|
||||
startupComplete = true;
|
||||
});
|
||||
|
||||
app.get('/health', (req, res, next) => {
|
||||
if (startupComplete) {
|
||||
return res.status(200).send('OK');
|
||||
|
|
|
|||
13
src/utils.js
13
src/utils.js
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
import path from 'path';
|
||||
import fs from 'node:fs';
|
||||
|
||||
import clone from 'clone';
|
||||
import glyphCompose from '@mapbox/glyph-pbf-composite';
|
||||
|
||||
|
|
@ -163,3 +162,15 @@ export const getFontsPbf = (
|
|||
|
||||
return Promise.all(queue).then((values) => glyphCompose.combine(values));
|
||||
};
|
||||
|
||||
export const isValidHttpUrl = (string) => {
|
||||
let url;
|
||||
|
||||
try {
|
||||
url = new URL(string);
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return url.protocol === 'http:' || url.protocol === 'https:';
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue