From a106fec3fc0a7ced06a0ccdda5dc71a88fc42502 Mon Sep 17 00:00:00 2001 From: Andrew Calcutt Date: Sat, 14 Oct 2023 20:33:42 -0400 Subject: [PATCH] fix: get data sources specified in style to work Signed-off-by: Andrew Calcutt --- src/serve_rendered.js | 224 ++++++++++++++++++++++-------------------- src/serve_style.js | 28 +++--- src/server.js | 51 +++++----- 3 files changed, 161 insertions(+), 142 deletions(-) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index e4290c7..47a3a10 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -1469,23 +1469,26 @@ export const serve_rendered = { for (const name of Object.keys(styleJSON.sources)) { let source_type; let source = styleJSON.sources[name]; - const url = source.url; + let url = source.url; + if (url) { + if (url.startsWith('{') && url.endsWith('}')) { + url = url.slice(1, -1); + } + if (url.startsWith('pmtiles://') || url.startsWith('mbtiles://')) { + // found pmtiles or mbtiles source, replace with info from local file + delete source.url; - if (url && (url.startsWith('pmtiles:') || url.startsWith('mbtiles:'))) { - // found pmtiles or mbtiles source, replace with info from local file - delete source.url; + let dataId = url.replace('pmtiles://', '').replace('mbtiles://', ''); + if (dataId.startsWith('{') && dataId.endsWith('}')) { + dataId = dataId.slice(1, -1); + } - let inputFile; - let dataId = url.replace('pmtiles://', '').replace('mbtiles://', ''); - const fromData = dataId.startsWith('{') && dataId.endsWith('}'); - - if (fromData) { - dataId = dataId.slice(1, -1); const mapsTo = (params.mapping || {})[dataId]; if (mapsTo) { dataId = mapsTo; } + let inputFile; const DataInfo = dataResolver(dataId); if (DataInfo.inputfile) { inputFile = DataInfo.inputfile; @@ -1494,109 +1497,114 @@ export const serve_rendered = { console.error(`ERROR: data "${inputFile}" not found!`); process.exit(1); } - } - if (!isValidHttpUrl(inputFile)) { - const inputFileStats = fs.statSync(inputFile); - if (!inputFileStats.isFile() || inputFileStats.size === 0) { - throw Error(`Not valid PMTiles file: "${inputFile}"`); - } - } - - if (source_type === 'pmtiles') { - map.sources[name] = PMtilesOpen(inputFile); - map.source_types[name] = 'pmtiles'; - const metadata = await GetPMtilesInfo(map.sources[name]); - - if (!repoobj.dataProjWGStoInternalWGS && metadata.proj4) { - // how to do this for multiple sources with different proj4 defs? - const to3857 = proj4('EPSG:3857'); - const toDataProj = proj4(metadata.proj4); - repoobj.dataProjWGStoInternalWGS = (xy) => - to3857.inverse(toDataProj.forward(xy)); - } - - const type = source.type; - metadata['extension'] = 'pmtiles'; - Object.assign(source, metadata); - source.type = type; - source.tiles = [ - // meta url which will be detected when requested - `pmtiles://${name}/{z}/{x}/{y}.${metadata.format || 'pbf'}`, - ]; - delete source.scheme; - - if ( - !attributionOverride && - source.attribution && - source.attribution.length > 0 - ) { - if (!tileJSON.attribution.includes(source.attribution)) { - if (tileJSON.attribution.length > 0) { - tileJSON.attribution += ' | '; - } - tileJSON.attribution += source.attribution; + if (!isValidHttpUrl(inputFile)) { + const inputFileStats = fs.statSync(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + throw Error(`Not valid PMTiles file: "${inputFile}"`); } } - } else { - queue.push( - new Promise((resolve, reject) => { - inputFile = path.resolve(options.paths.mbtiles, inputFile); - const inputFileStats = fs.statSync(inputFile); - if (!inputFileStats.isFile() || inputFileStats.size === 0) { - throw Error(`Not valid MBTiles file: "${inputFile}"`); + + if (source_type === 'pmtiles') { + map.sources[name] = PMtilesOpen(inputFile); + map.source_types[name] = 'pmtiles'; + const metadata = await GetPMtilesInfo(map.sources[name]); + + if (!repoobj.dataProjWGStoInternalWGS && metadata.proj4) { + // how to do this for multiple sources with different proj4 defs? + const to3857 = proj4('EPSG:3857'); + const toDataProj = proj4(metadata.proj4); + repoobj.dataProjWGStoInternalWGS = (xy) => + to3857.inverse(toDataProj.forward(xy)); + } + + const type = source.type; + metadata['extension'] = 'pmtiles'; + Object.assign(source, metadata); + source.type = type; + source.tiles = [ + // meta url which will be detected when requested + `pmtiles://${name}/{z}/{x}/{y}.${metadata.format || 'pbf'}`, + ]; + delete source.scheme; + + if ( + !attributionOverride && + source.attribution && + source.attribution.length > 0 + ) { + if (!tileJSON.attribution.includes(source.attribution)) { + if (tileJSON.attribution.length > 0) { + tileJSON.attribution += ' | '; + } + tileJSON.attribution += source.attribution; } - map.sources[name] = new MBTiles(inputFile + '?mode=ro', (err) => { - map.sources[name].getInfo((err, info) => { - if (err) { - console.error(err); - return; - } - map.source_types[name] = 'mbtiles'; - - if (!repoobj.dataProjWGStoInternalWGS && info.proj4) { - // how to do this for multiple sources with different proj4 defs? - const to3857 = proj4('EPSG:3857'); - const toDataProj = proj4(info.proj4); - repoobj.dataProjWGStoInternalWGS = (xy) => - to3857.inverse(toDataProj.forward(xy)); - } - - const type = source.type; - info['extension'] = 'mbtiles'; - Object.assign(source, info); - source.type = type; - source.tiles = [ - // meta url which will be detected when requested - `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`, - ]; - delete source.scheme; - - if (options.dataDecoratorFunc) { - source = options.dataDecoratorFunc( - name, - 'tilejson', - source, - ); - } - - if ( - !attributionOverride && - source.attribution && - source.attribution.length > 0 - ) { - if (!tileJSON.attribution.includes(source.attribution)) { - if (tileJSON.attribution.length > 0) { - tileJSON.attribution += ' | '; + } + } else { + queue.push( + new Promise((resolve, reject) => { + inputFile = path.resolve(options.paths.mbtiles, inputFile); + const inputFileStats = fs.statSync(inputFile); + if (!inputFileStats.isFile() || inputFileStats.size === 0) { + throw Error(`Not valid MBTiles file: "${inputFile}"`); + } + map.sources[name] = new MBTiles( + inputFile + '?mode=ro', + (err) => { + map.sources[name].getInfo((err, info) => { + if (err) { + console.error(err); + return; } - tileJSON.attribution += source.attribution; - } - } - resolve(); - }); - }); - }), - ); + map.source_types[name] = 'mbtiles'; + + if (!repoobj.dataProjWGStoInternalWGS && info.proj4) { + // how to do this for multiple sources with different proj4 defs? + const to3857 = proj4('EPSG:3857'); + const toDataProj = proj4(info.proj4); + repoobj.dataProjWGStoInternalWGS = (xy) => + to3857.inverse(toDataProj.forward(xy)); + } + + const type = source.type; + info['extension'] = 'mbtiles'; + Object.assign(source, info); + source.type = type; + source.tiles = [ + // meta url which will be detected when requested + `mbtiles://${name}/{z}/{x}/{y}.${info.format || 'pbf'}`, + ]; + delete source.scheme; + + if (options.dataDecoratorFunc) { + source = options.dataDecoratorFunc( + name, + 'tilejson', + source, + ); + } + + if ( + !attributionOverride && + source.attribution && + source.attribution.length > 0 + ) { + if ( + !tileJSON.attribution.includes(source.attribution) + ) { + if (tileJSON.attribution.length > 0) { + tileJSON.attribution += ' | '; + } + tileJSON.attribution += source.attribution; + } + } + resolve(); + }); + }, + ); + }), + ); + } } } } diff --git a/src/serve_style.js b/src/serve_style.js index efc77de..3c982f2 100644 --- a/src/serve_style.js +++ b/src/serve_style.js @@ -110,26 +110,30 @@ export const serve_style = { for (const name of Object.keys(styleJSON.sources)) { const source = styleJSON.sources[name]; - const url = source.url; + let url = source.url; + if (url) { + if (url.startsWith('{') && url.endsWith('}')) { + url = url.slice(1, -1); + } + if (url.startsWith('pmtiles://') || url.startsWith('mbtiles://')) { + const protocol = url.split(':')[0]; - if (url && (url.startsWith('pmtiles:') || url.startsWith('mbtiles:'))) { - const protocol = url.split(':')[0]; - let dataId = url.replace('pmtiles://', '').replace('mbtiles://', ''); + let dataId = url.replace('pmtiles://', '').replace('mbtiles://', ''); + if (dataId.startsWith('{') && dataId.endsWith('}')) { + dataId = dataId.slice(1, -1); + } - const fromData = dataId.startsWith('{') && dataId.endsWith('}'); - if (fromData) { - dataId = dataId.slice(1, -1); const mapsTo = (params.mapping || {})[dataId]; if (mapsTo) { dataId = mapsTo; } - } - const identifier = reportTiles(dataId, fromData, protocol); - if (!identifier) { - return false; + const identifier = reportTiles(dataId, protocol); + if (!identifier) { + return false; + } + source.url = `local://data/${identifier}.json`; } - source.url = `local://data/${identifier}.json`; } } diff --git a/src/server.js b/src/server.js index 68bd836..87a476b 100644 --- a/src/server.js +++ b/src/server.js @@ -183,32 +183,38 @@ function start(opts) { item, id, opts.publicUrl, - (dataId, fromData, protocol) => { + (StyleSourceId, protocol) => { let dataItemId; for (const id of Object.keys(data)) { - if (fromData) { - if (id === dataId) { - dataItemId = id; - } + if (id === StyleSourceId) { + // Style id was found in data ids, return that id + dataItemId = id; } else { const fileType = Object.keys(data[id])[0]; - if (data[id][fileType] === dataId) { + if (data[id][fileType] === StyleSourceId) { + // Style id was found in data filename, return the id that filename belong to dataItemId = id; } } } if (dataItemId) { - // input files exists in the data config + // input files exists in the data config, return found id return dataItemId; } else { - if (fromData || !allowMoreData) { + if (!allowMoreData) { console.log( - `ERROR: style "${item.style}" using unknown file "${dataId}"! Skipping...`, + `ERROR: style "${item.style}" using unknown file "${StyleSourceId}"! Skipping...`, ); return undefined; } else { - let id = dataId.substr(0, dataId.lastIndexOf('.')) || dataId; - data[id][protocol] = dataId; + let id = StyleSourceId.replace(/^.*\/(.*)$/, '$1'); // Remove url path up to last backslash, if it exists + id = id.substr(0, StyleSourceId.lastIndexOf('.')) || id; // Remove extension, if it exists + while (data[id]) id += '_'; //if the data source id already exists, add a "_" untill it doesn't + //Add the new data source to the data array. + data[id] = { + [protocol]: StyleSourceId, + }; + return id; } } @@ -229,22 +235,23 @@ function start(opts) { item, id, opts.publicUrl, - (dataId) => { + (StyleSourceId) => { let fileType; let inputFile; for (const id of Object.keys(data)) { - if (id === dataId) { - fileType = Object.keys(data[id])[0]; - if (isValidHttpUrl(data[id][fileType])) { - inputFile = data[id][fileType]; - } else { - inputFile = path.resolve( - options.paths[fileType], - data[id][fileType], - ); - } + fileType = Object.keys(data[id])[0]; + if (StyleSourceId == id) { + inputFile = data[id][fileType]; + break; + } else if (data[id][fileType] == StyleSourceId) { + inputFile = data[id][fileType]; + break; } } + if (!isValidHttpUrl(inputFile)) { + inputFile = path.resolve(options.paths[fileType], inputFile); + } + return { inputfile: inputFile, filetype: fileType }; }, ),