Fabio/tileserver-gl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: use cust source so pmtiles func can be used

Browse source 
Custom source only loads requested bytes instead of the whole file like FileAPISource

Signed-off-by: Andrew Calcutt <acalcutt@techidiots.net>
This commit is contained in:
Andrew Calcutt 2023-10-09 11:34:33 -04:00
parent 445d03d626
commit 6ff506af99
1 changed files with 28 additions and 178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

206
src/pmtiles_adapter.js
View file

@ -1,24 +1,24 @@
import fs from 'node:fs'; import fs from 'node:fs';
import zlib from 'zlib';
import PMTiles from 'pmtiles'; import PMTiles from 'pmtiles';
export const GetPMtilesHeader = async (pmtilesFile) => { var PMTilesLocalSource = class {
var buffer = await ReadBytes(pmtilesFile, 0, 127); constructor(file) {
const header = PMTiles.bytesToHeader(buffer, undefined); this.file = file;
return header; }
getKey() {
return this.file.name;
}
async getBytes(offset, length) {
const blob = await ReadFileBytes(this.file, offset, length);
return { data: blob };
}
}; };
export const GetPMtilesInfo = async (pmtilesFile) => { export const GetPMtilesInfo = async (pmtilesFile) => {
//Get metadata from pmtiles file const source = new PMTilesLocalSource(pmtilesFile);
var header = await GetPMtilesHeader(pmtilesFile); const pmtiles = new PMTiles.PMTiles(source);
const metadataBytes = await ReadBytes( const header = await pmtiles.getHeader();
pmtilesFile, const metadata = await pmtiles.getMetadata()
header.jsonMetadataOffset,
header.jsonMetadataLength,
);
const metadataDecomp = await GetPMtilesDecompress(header, metadataBytes);
const dec = new TextDecoder('utf-8');
const metadata = JSON.parse(dec.decode(metadataDecomp));
//Add missing metadata from header //Add missing metadata from header
const bounds = [header.minLon, header.minLat, header.maxLon, header.maxLat]; const bounds = [header.minLon, header.minLat, header.maxLon, header.maxLat];
@ -34,45 +34,13 @@ export const GetPMtilesInfo = async (pmtilesFile) => {
}; };
export const GetPMtilesTile = async (pmtilesFile, z, x, y) => { export const GetPMtilesTile = async (pmtilesFile, z, x, y) => {
const tile_id = PMTiles.zxyToTileId(z, x, y); const source = new PMTilesLocalSource(pmtilesFile);
const header = await GetPMtilesHeader(pmtilesFile); const pmtiles = new PMTiles.PMTiles(source);
const header = await pmtiles.getHeader();
if (z < header.minZoom || z > header.maxZoom) { const TileType = GetPmtilesTileType(header.tileType);
return undefined; let zxyTile = await pmtiles.getZxy(z, x, y);
} if(zxyTile.data !== undefined){zxyTile = ArrayBufferToBuffer(zxyTile.data);} else {zxyTile = undefined}
return { data: zxyTile, header: TileType.header };
let rootDirectoryOffset = header.rootDirectoryOffset;
let rootDirectoryLength = header.rootDirectoryLength;
for (let depth = 0; depth <= 3; depth++) {
const RootDirectoryBytes = await ReadBytes(
pmtilesFile,
rootDirectoryOffset,
rootDirectoryLength,
);
const RootDirectoryBytesaDecomp = await GetPMtilesDecompress(
header,
RootDirectoryBytes,
);
const Directory = deserializeIndex(RootDirectoryBytesaDecomp);
const entry = PMTiles.findTile(Directory, tile_id);
if (entry) {
if (entry.runLength > 0) {
const EntryBytesArrayBuff = await ReadBytes(
pmtilesFile,
header.tileDataOffset + entry.offset,
entry.length,
);
const EntryBytes = ArrayBufferToBuffer(EntryBytesArrayBuff);
const EntryTileType = GetPmtilesTileType(header.tileType);
return { data: EntryBytes, header: EntryTileType.header };
} else {
rootDirectoryOffset = header.leafDirectoryOffset + entry.offset;
rootDirectoryLength = entry.length;
}
} else {
return undefined;
}
}
}; };
/** /**
@ -126,16 +94,16 @@ function BufferToArrayBuffer(buffer) {
* *
* @param ab * @param ab
*/ */
function ArrayBufferToBuffer(ab) { function ArrayBufferToBuffer(array_buffer) {
var buffer = Buffer.alloc(ab.byteLength); var buffer = Buffer.alloc(array_buffer.byteLength);
var view = new Uint8Array(ab); var view = new Uint8Array(array_buffer);
for (var i = 0; i < buffer.length; ++i) { for (var i = 0; i < buffer.length; ++i) {
buffer[i] = view[i]; buffer[i] = view[i];
} }
return buffer; return buffer;
} }
const ReadBytes = async (filePath, offset, size) => { const ReadFileBytes = async (filePath, offset, size) => {
const sharedBuffer = Buffer.alloc(size); const sharedBuffer = Buffer.alloc(size);
const fd = fs.openSync(filePath); // file descriptor const fd = fs.openSync(filePath); // file descriptor
const stats = fs.fstatSync(fd); // file details const stats = fs.fstatSync(fd); // file details
@ -143,7 +111,7 @@ const ReadBytes = async (filePath, offset, size) => {
for (let i = 0; i < size; i++) { for (let i = 0; i < size; i++) {
let postion = offset + i; let postion = offset + i;
await ReadFileBytes(fd, sharedBuffer, postion); await ReadBytes(fd, sharedBuffer, postion);
bytesRead = (i + 1) * size; bytesRead = (i + 1) * size;
if (bytesRead > stats.size) { if (bytesRead > stats.size) {
// When we reach the end of file, // When we reach the end of file,
@ -165,7 +133,7 @@ const ReadBytes = async (filePath, offset, size) => {
* @param sharedBuffer * @param sharedBuffer
* @param offset * @param offset
*/ */
function ReadFileBytes(fd, sharedBuffer, offset) { function ReadBytes(fd, sharedBuffer, offset) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
fs.read(fd, sharedBuffer, 0, sharedBuffer.length, offset, (err) => { fs.read(fd, sharedBuffer, 0, sharedBuffer.length, offset, (err) => {
if (err) { if (err) {
@ -175,121 +143,3 @@ function ReadFileBytes(fd, sharedBuffer, offset) {
}); });
}); });
} }
export const GetPMtilesDecompress = async (header, buffer) => {
const compression = header.internalCompression;
var decompressed;
if (
compression === PMTiles.Compression.None ||
compression === PMTiles.Compression.Unknown
) {
decompressed = buffer;
} else if (compression === PMTiles.Compression.Gzip) {
decompressed = zlib.unzipSync(buffer);
} else {
throw Error('Compression method not supported');
}
return decompressed;
};
/**
*
* @param low
* @param high
*/
function toNum(low, high) {
return (high >>> 0) * 0x100000000 + (low >>> 0);
}
/**
*
* @param l
* @param p
*/
function readVarintRemainder(l, p) {
const buf = p.buf;
let h, b;
b = buf[p.pos++];
h = (b & 0x70) >> 4;
if (b < 0x80) return toNum(l, h);
b = buf[p.pos++];
h |= (b & 0x7f) << 3;
if (b < 0x80) return toNum(l, h);
b = buf[p.pos++];
h |= (b & 0x7f) << 10;
if (b < 0x80) return toNum(l, h);
b = buf[p.pos++];
h |= (b & 0x7f) << 17;
if (b < 0x80) return toNum(l, h);
b = buf[p.pos++];
h |= (b & 0x7f) << 24;
if (b < 0x80) return toNum(l, h);
b = buf[p.pos++];
h |= (b & 0x01) << 31;
if (b < 0x80) return toNum(l, h);
throw new Error('Expected varint not more than 10 bytes');
}
/**
*
* @param p
*/
export function readVarint(p) {
const buf = p.buf;
let val, b;
b = buf[p.pos++];
val = b & 0x7f;
if (b < 0x80) return val;
b = buf[p.pos++];
val |= (b & 0x7f) << 7;
if (b < 0x80) return val;
b = buf[p.pos++];
val |= (b & 0x7f) << 14;
if (b < 0x80) return val;
b = buf[p.pos++];
val |= (b & 0x7f) << 21;
if (b < 0x80) return val;
b = buf[p.pos];
val |= (b & 0x0f) << 28;
return readVarintRemainder(val, p);
}
/**
*
* @param buffer
*/
function deserializeIndex(buffer) {
const p = { buf: new Uint8Array(buffer), pos: 0 };
const numEntries = readVarint(p);
var entries = [];
let lastId = 0;
for (let i = 0; i < numEntries; i++) {
const v = readVarint(p);
entries.push({ tileId: lastId + v, offset: 0, length: 0, runLength: 1 });
lastId += v;
}
for (let i = 0; i < numEntries; i++) {
entries[i].runLength = readVarint(p);
}
for (let i = 0; i < numEntries; i++) {
entries[i].length = readVarint(p);
}
for (let i = 0; i < numEntries; i++) {
const v = readVarint(p);
if (v === 0 && i > 0) {
entries[i].offset = entries[i - 1].offset + entries[i - 1].length;
} else {
entries[i].offset = v - 1;
}
}
return entries;
}