brouter/misc/pbfparser/BPbfBlobDecoder.java

package btools.mapcreator;

import com.google.protobuf.InvalidProtocolBufferException;
import org.openstreetmap.osmosis.osmbinary.Fileformat;
import org.openstreetmap.osmosis.osmbinary.Osmformat;
import btools.util.LongList;

import java.io.IOException;
import java.util.*;
import java.util.logging.Level;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;

/**
 * Converts PBF block data into decoded entities ready to be passed into an Osmosis pipeline. This
 * class is designed to be passed into a pool of worker threads to allow multi-threaded decoding.
 * <p/>
 * @author Brett Henderson
 */
public class BPbfBlobDecoder
{
    private String blobType;
    private byte[] rawBlob;

    private OsmParser parser;

    /**
     * Creates a new instance.
     * <p/>
     * @param blobType The type of blob.
     * @param rawBlob The raw data of the blob.
     * @param listener The listener for receiving decoding results.
     */
    public BPbfBlobDecoder( String blobType, byte[] rawBlob, OsmParser parser )
    {
        this.blobType = blobType;
        this.rawBlob = rawBlob;
        this.parser = parser;
    }

    public void process() throws Exception
    {
            if ("OSMHeader".equals(blobType))
            {
                processOsmHeader(readBlobContent());

            } else if ("OSMData".equals(blobType))
            {
                processOsmPrimitives(readBlobContent());

            } else
            {
              System.out.println("Skipping unrecognised blob type " + blobType);
            }
    }

    private byte[] readBlobContent() throws IOException
    {
        Fileformat.Blob blob = Fileformat.Blob.parseFrom(rawBlob);
        byte[] blobData;

        if (blob.hasRaw())
        {
            blobData = blob.getRaw().toByteArray();
        } else if (blob.hasZlibData())
        {
            Inflater inflater = new Inflater();
            inflater.setInput(blob.getZlibData().toByteArray());
            blobData = new byte[blob.getRawSize()];
            try
            {
                inflater.inflate(blobData);
            } catch (DataFormatException e)
            {
                throw new RuntimeException("Unable to decompress PBF blob.", e);
            }
            if (!inflater.finished())
            {
                throw new RuntimeException("PBF blob contains incomplete compressed data.");
            }
        } else
        {
            throw new RuntimeException("PBF blob uses unsupported compression, only raw or zlib may be used.");
        }

        return blobData;
    }

    private void processOsmHeader( byte[] data ) throws InvalidProtocolBufferException
    {
        Osmformat.HeaderBlock header = Osmformat.HeaderBlock.parseFrom(data);

        // Build the list of active and unsupported features in the file.
        List<String> supportedFeatures = Arrays.asList("OsmSchema-V0.6", "DenseNodes");
        List<String> activeFeatures = new ArrayList<String>();
        List<String> unsupportedFeatures = new ArrayList<String>();
        for (String feature : header.getRequiredFeaturesList())
        {
            if (supportedFeatures.contains(feature))
            {
                activeFeatures.add(feature);
            } else
            {
                unsupportedFeatures.add(feature);
            }
        }

        // We can't continue if there are any unsupported features. We wait
        // until now so that we can display all unsupported features instead of
        // just the first one we encounter.
        if (unsupportedFeatures.size() > 0)
        {
            throw new RuntimeException("PBF file contains unsupported features " + unsupportedFeatures);
        }

    }

    private Map<String, String> buildTags( List<Integer> keys, List<Integer> values, BPbfFieldDecoder fieldDecoder )
    {

        Iterator<Integer> keyIterator = keys.iterator();
        Iterator<Integer> valueIterator = values.iterator();
        if (keyIterator.hasNext())
        {
            Map<String, String> tags = new HashMap<String, String>();
            while (keyIterator.hasNext())
            {
                String key = fieldDecoder.decodeString(keyIterator.next());
                String value = fieldDecoder.decodeString(valueIterator.next());
                tags.put(key, value);
            }
            return tags;
        }
        return null;
    }

    private void processNodes( List<Osmformat.Node> nodes, BPbfFieldDecoder fieldDecoder )
    {
        for (Osmformat.Node node : nodes)
        {
            Map<String, String> tags = buildTags(node.getKeysList(), node.getValsList(), fieldDecoder);

            parser.addNode( node.getId(), tags, fieldDecoder.decodeLatitude(node
                    .getLat()), fieldDecoder.decodeLatitude(node.getLon()));
        }
    }

    private void processNodes( Osmformat.DenseNodes nodes, BPbfFieldDecoder fieldDecoder )
    {
        List<Long> idList = nodes.getIdList();
        List<Long> latList = nodes.getLatList();
        List<Long> lonList = nodes.getLonList();

        Iterator<Integer> keysValuesIterator = nodes.getKeysValsList().iterator();

        long nodeId = 0;
        long latitude = 0;
        long longitude = 0;

        for (int i = 0; i < idList.size(); i++)
        {
            // Delta decode node fields.
            nodeId += idList.get(i);
            latitude += latList.get(i);
            longitude += lonList.get(i);

            // Build the tags. The key and value string indexes are sequential
            // in the same PBF array. Each set of tags is delimited by an index
            // with a value of 0.
            Map<String, String> tags = null;
            while (keysValuesIterator.hasNext())
            {
                int keyIndex = keysValuesIterator.next();
                if (keyIndex == 0)
                {
                    break;
                }
                int valueIndex = keysValuesIterator.next();

                if (tags == null)
                {
                    tags = new HashMap<String, String>();
                }

                tags.put(fieldDecoder.decodeString(keyIndex), fieldDecoder.decodeString(valueIndex));
            }

            parser.addNode( nodeId, tags, ((double) latitude) / 10000000, ((double) longitude) / 10000000);
        }
    }

    private void processWays( List<Osmformat.Way> ways, BPbfFieldDecoder fieldDecoder )
    {
        for (Osmformat.Way way : ways)
        {
            Map<String, String> tags = buildTags(way.getKeysList(), way.getValsList(), fieldDecoder);

            // Build up the list of way nodes for the way. The node ids are
            // delta encoded meaning that each id is stored as a delta against
            // the previous one.
            long nodeId = 0;
            LongList wayNodes = new LongList( 16 );
            for (long nodeIdOffset : way.getRefsList())
            {
                nodeId += nodeIdOffset;
                wayNodes.add(nodeId);
            }

            parser.addWay( way.getId(), tags, wayNodes );
        }
    }

    private long fromWid;
    private long toWid;
    private long viaNid;

    private LongList buildRelationMembers(
            List<Long> memberIds, List<Integer> memberRoles, List<Osmformat.Relation.MemberType> memberTypes,
            BPbfFieldDecoder fieldDecoder )
    {
        LongList wayIds = new LongList( 16 );

        fromWid = toWid = viaNid = 0;

        Iterator<Long> memberIdIterator = memberIds.iterator();
        Iterator<Integer> memberRoleIterator = memberRoles.iterator();
        Iterator<Osmformat.Relation.MemberType> memberTypeIterator = memberTypes.iterator();

        // Build up the list of relation members for the way. The member ids are
        // delta encoded meaning that each id is stored as a delta against
        // the previous one.
        long refId = 0;
        while (memberIdIterator.hasNext())
        {
            Osmformat.Relation.MemberType memberType = memberTypeIterator.next();
            refId += memberIdIterator.next();

            String role = fieldDecoder.decodeString( memberRoleIterator.next() );

            if ( memberType == Osmformat.Relation.MemberType.WAY ) // currently just waymembers
            {
              wayIds.add( refId );
              if ( "from".equals( role ) ) fromWid = refId;
              if ( "to".equals( role ) ) toWid = refId;
            }
            if ( memberType == Osmformat.Relation.MemberType.NODE ) // currently just waymembers
            {
              if ( "via".equals( role ) ) viaNid = refId;
            }
        }
        return wayIds;
    }

    private void processRelations( List<Osmformat.Relation> relations, BPbfFieldDecoder fieldDecoder )
    {
        for (Osmformat.Relation relation : relations)
        {
            Map<String, String> tags = buildTags(relation.getKeysList(), relation.getValsList(), fieldDecoder);

            LongList wayIds = buildRelationMembers( relation.getMemidsList(), relation.getRolesSidList(),
                    relation.getTypesList(), fieldDecoder);

            parser.addRelation( relation.getId(), tags, wayIds, fromWid, toWid, viaNid );
        }
    }

    private void processOsmPrimitives( byte[] data ) throws InvalidProtocolBufferException
    {
        Osmformat.PrimitiveBlock block = Osmformat.PrimitiveBlock.parseFrom(data);
        BPbfFieldDecoder fieldDecoder = new BPbfFieldDecoder(block);

        for (Osmformat.PrimitiveGroup primitiveGroup : block.getPrimitivegroupList())
        {
            processNodes(primitiveGroup.getDense(), fieldDecoder);
            processNodes(primitiveGroup.getNodesList(), fieldDecoder);
            processWays(primitiveGroup.getWaysList(), fieldDecoder);
            processRelations(primitiveGroup.getRelationsList(), fieldDecoder);
        }
    }

}