From a5addd567d627a3ac9e9a9b740e3907733d952b0 Mon Sep 17 00:00:00 2001 From: David Weber | geOps Date: Sun, 31 Dec 2023 13:38:20 +0100 Subject: [PATCH 1/5] fix: add remove function to serve_data Signed-off-by: David Weber | geOps --- src/serve_data.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/serve_data.js b/src/serve_data.js index 4a95b1f..a50c507 100644 --- a/src/serve_data.js +++ b/src/serve_data.js @@ -189,6 +189,9 @@ export const serve_data = { return app; }, + remove: (repo, id) => { + delete repo[id]; + }, add: async (options, repo, params, id, publicUrl) => { let inputFile; let inputType; From ee1372fd552d7217a928d592cf190243ecc79a3b Mon Sep 17 00:00:00 2001 From: David Weber | geOps Date: Sun, 31 Dec 2023 13:40:27 +0100 Subject: [PATCH 2/5] fix: explicitly delete all sources when removing rendered GC sometimes cannot do it himself. Maybe a self-referencing problem Signed-off-by: David Weber | geOps --- src/serve_rendered.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index ec590b6..21ee2cf 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -1250,6 +1250,9 @@ export const serve_rendered = { item.map.renderersStatic.forEach((pool) => { pool.close(); }); + Object.entries(item.map.sources).forEach(([key, source]) => { + delete item.map.sources[key]; + }); } delete repo[id]; }, From a088080e27e080bda5dc7b3575cfda4be574d92e Mon Sep 17 00:00:00 2001 From: David Weber | geOps Date: Sun, 31 Dec 2023 13:41:57 +0100 Subject: [PATCH 3/5] fix: store styleFile in serve_rendered necessary to implement hot mbtiles reloading Signed-off-by: David Weber | geOps --- src/serve_rendered.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/serve_rendered.js b/src/serve_rendered.js index 21ee2cf..2f6d71a 100644 --- a/src/serve_rendered.js +++ b/src/serve_rendered.js @@ -847,6 +847,7 @@ export const serve_rendered = { renderersStatic: [], sources: {}, sourceTypes: {}, + styleFile: '', }; let styleJSON; @@ -1023,6 +1024,7 @@ export const serve_rendered = { }; const styleFile = params.style; + map.styleFile = styleFile; const styleJSONPath = path.resolve(options.paths.styles, styleFile); try { styleJSON = JSON.parse(fs.readFileSync(styleJSONPath)); From 7e6c6f31a72f6ee9729a78e543c1d4cd15d10fdc Mon Sep 17 00:00:00 2001 From: David Weber | geOps Date: Sun, 31 Dec 2023 14:03:00 +0100 Subject: [PATCH 4/5] feat: implement watching mbtiles and hot-reloading this will watch for changes in the mbtiles and hot-reload it. To keep the implementation simple, the whole renderer is restartet. This will create a small downtime. It is very important to replace files atomically --- docs/config.rst | 8 +++++++ src/server.js | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/docs/config.rst b/docs/config.rst index 9078d99..1114fec 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -32,6 +32,7 @@ Example: "pbfAlias": "pbf", "serveAllFonts": false, "serveAllStyles": false, + "watchMbtiles": false, "serveStaticMaps": true, "allowRemoteMarkerIcons": true, "allowInlineMarkerImages": true, @@ -152,6 +153,13 @@ If this option is enabled, all the styles from the ``paths.styles`` will be serv The process will also watch for changes in this directory and remove/add more styles dynamically. It is recommended to also use the ``serveAllFonts`` option when using this option. +``watchMbtiles`` +------------------------ + +If this option is enabled, all the opened Mbtiles are watched for changes and automatically reloaded. +The new data is then severed immediately. There is a small downtime for rendered endpoints. +Mbtiles have to be replaced atomically. i.e. moving the file from the same filesystem. Modifying an existing file will crash the server. + ``serveStaticMaps`` ------------------------ diff --git a/src/server.js b/src/server.js index 2b382a0..61122dd 100644 --- a/src/server.js +++ b/src/server.js @@ -292,6 +292,69 @@ function start(opts) { startupPromises.push( serve_data.add(options, serving.data, item, id, opts.publicUrl), ); + + if (options.watchMbtiles) { + console.log(`Watching Mbtile "${item.mbtiles}" for changes...`); + + const watcher = chokidar.watch( + path.join(options.paths.mbtiles, item.mbtiles), + { + ignoreInitial: true, + // wait 10 seconds after the last change before updating. Otherwise, cases where a file is constantly replaced + // will create race conditions and can crash the server + awaitWriteFinish: { stabilityThreshold: 10000 }, + }, + ); + + watcher.on('all', (eventType, filename) => { + if (filename) { + if (eventType === 'add' || eventType === 'change') { + console.log(`MBTiles "${filename}" changed, updating...`); + + serve_data.remove(serving.data, id); + let newItem = { + mbtiles: filename, + }; + serve_data.add(options, serving.data, newItem, id, opts.publicUrl); + + if (!isLight) { + Object.entries(serving.rendered).forEach( + ([serving_key, serving_value]) => { + // check if source is used in serving + Object.values(serving_value.map.sources).forEach( + (source_value) => { + const newFileInode = fs.statSync(filename).ino; + // we check if the filename is the same and the inode has changed + // the inode check is necessary because it could be that multiple mbtiles + // were changed and we already changed to the new file. This can lead to race-conditions + if ( + source_value.filename === filename && + source_value._stat.ino !== newFileInode + ) { + // remove from serving and add back + serve_style.remove(serving.styles, serving_key); + serve_rendered.remove(serving.rendered, serving_key); + + const item = { + style: serving_value.map.styleFile, + }; + addStyle(serving_key, item, false, false); + } + }, + ); + }, + ); + } + } + // we intentionally don't handle the 'unlink' event here. If the file is deleted, the file descriptor is still valid, + // everything will continue to work + } + }); + + watcher.on('error', (error) => { + console.error(`Failed to watch file: ${error}`); + }); + } } if (options.serveAllStyles) { From d5765328ec46f734db4df092afb1b0c6c82794e7 Mon Sep 17 00:00:00 2001 From: David Weber | geOps Date: Sun, 31 Dec 2023 14:04:40 +0100 Subject: [PATCH 5/5] feat: enable watchMbtiles for input file mode Signed-off-by: David Weber | geOps --- src/main.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.js b/src/main.js index 5077ba0..32fbd45 100644 --- a/src/main.js +++ b/src/main.js @@ -109,6 +109,7 @@ const startWithInputFile = async (inputFile) => { const config = { options: { + watchMbtiles: true, paths: { root: styleDir, fonts: 'fonts',