Why Don't support partitioned vertex while I using Janus-hadoop


spirit...@...
 

17:52:45,251  INFO RemoteActorRefProvider$RemotingTerminator:74 - Remote daemon shut down; proceeding with flushing remote transports.
org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 3, 10.200.48.158): java.lang.IllegalStateException: Read partitioned vertex (ID=8202), but partitioned vertex filtering is disabled.
at com.google.common.base.Preconditions.checkState(Preconditions.java:176)
at org.janusgraph.hadoop.formats.util.JanusGraphVertexDeserializer.readHadoopVertex(JanusGraphVertexDeserializer.java:84)
at org.janusgraph.hadoop.formats.util.GiraphRecordReader.nextKeyValue(GiraphRecordReader.java:60)
at org.apache.spark.rdd.NewHadoopRDD$$anon$1.hasNext(NewHadoopRDD.scala:168)
at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)
at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)
at scala.collection.convert.Wrappers$IteratorWrapper.hasNext(Wrappers.scala:29)
at org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils$4.advance(IteratorUtils.java:298)
at org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils$4.hasNext(IteratorUtils.java:269)
at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:41)
at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:126)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
at org.apache.spark.scheduler.Task.run(Task.scala:89)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:214)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

I find out the code in JanusGraph, the following
 // Read a single row from the edgestore and create a TinkerVertex corresponding to the row
// The neighboring vertices are represented by DetachedVertex instances
public TinkerVertex readHadoopVertex(final StaticBuffer key, Iterable<Entry> entries) {

// Convert key to a vertex ID
final long vertexId = idManager.getKeyID(key);
Preconditions.checkArgument(vertexId > 0);

// Partitioned vertex handling
if (idManager.isPartitionedVertex(vertexId)) {
Preconditions.checkState(setup.getFilterPartitionedVertices(),
"Read partitioned vertex (ID=%s), but partitioned vertex filtering is disabled.", vertexId);
log.debug("Skipping partitioned vertex with ID {}", vertexId);
return null;
}

// Create TinkerVertex
TinkerGraph tg = TinkerGraph.open();

boolean foundVertexState = !verifyVertexExistence;

TinkerVertex tv = null;

// Iterate over edgestore columns to find the vertex's label relation
for (final Entry data : entries) {
RelationReader relationReader = setup.getRelationReader(vertexId);
final RelationCache relation = relationReader.parseRelation(data, false, typeManager);
if (systemTypes.isVertexLabelSystemType(relation.typeId)) {
// Found vertex Label
long vertexLabelId = relation.getOtherVertexId();
VertexLabel vl = typeManager.getExistingVertexLabel(vertexLabelId);
// Create TinkerVertex with this label
//tv = (TinkerVertex)tg.addVertex(T.label, vl.label(), T.id, vertexId);
tv = getOrCreateVertex(vertexId, vl.name(), tg);
}
}

// Added this following testing
if (null == tv) {
//tv = (TinkerVertex)tg.addVertex(T.id, vertexId);
tv = getOrCreateVertex(vertexId, null, tg);
}

Preconditions.checkState(null != tv, "Unable to determine vertex label for vertex with ID %s", vertexId);

// Iterate over and decode edgestore columns (relations) on this vertex
for (final Entry data : entries) {
try {
RelationReader relationReader = setup.getRelationReader(vertexId);
final RelationCache relation = relationReader.parseRelation(data, false, typeManager);
if (systemTypes.isVertexExistsSystemType(relation.typeId)) {
foundVertexState = true;
}

if (systemTypes.isSystemType(relation.typeId)) continue; //Ignore system types
final RelationType type = typeManager.getExistingRelationType(relation.typeId);
if (((InternalRelationType)type).isInvisibleType()) continue; //Ignore hidden types

// Decode and create the relation (edge or property)
if (type.isPropertyKey()) {
// Decode property
Object value = relation.getValue();
Preconditions.checkNotNull(value);
VertexProperty.Cardinality card = getPropertyKeyCardinality(type.name());
tv.property(card, type.name(), value, T.id, relation.relationId);
} else {
assert type.isEdgeLabel();

// Partitioned vertex handling
if (idManager.isPartitionedVertex(relation.getOtherVertexId())) {
Preconditions.checkState(setup.getFilterPartitionedVertices(),
"Read edge incident on a partitioned vertex, but partitioned vertex filtering is disabled. " +
"Relation ID: %s. This vertex ID: %s. Other vertex ID: %s. Edge label: %s.",
relation.relationId, vertexId, relation.getOtherVertexId(), type.name());
log.debug("Skipping edge with ID {} incident on partitioned vertex with ID {} (and nonpartitioned vertex with ID {})",
relation.relationId, relation.getOtherVertexId(), vertexId);
continue;
}

// Decode edge
TinkerEdge te;

// We don't know the label of the other vertex, but one must be provided
TinkerVertex adjacentVertex = getOrCreateVertex(relation.getOtherVertexId(), null, tg);

// handle self-loop edges
if (tv.equals(adjacentVertex) && isLoopAdded(tv, type.name())) {
continue;
}

if (relation.direction.equals(Direction.IN)) {
te = (TinkerEdge)adjacentVertex.addEdge(type.name(), tv, T.id, relation.relationId);
} else if (relation.direction.equals(Direction.OUT)) {
te = (TinkerEdge)tv.addEdge(type.name(), adjacentVertex, T.id, relation.relationId);
} else {
throw new RuntimeException("Direction.BOTH is not supported");
}

if (relation.hasProperties()) {
// Load relation properties
for (final LongObjectCursor<Object> next : relation) {
assert next.value != null;
RelationType rt = typeManager.getExistingRelationType(next.key);
if (rt.isPropertyKey()) {
// PropertyKey pkey = (PropertyKey)vertex.getTypeManager().getPropertyKey(rt.name());
// log.debug("Retrieved key {} for name \"{}\"", pkey, rt.name());
// frel.property(pkey.label(), next.value);
te.property(rt.name(), next.value);
} else {
throw new RuntimeException("Metaedges are not supported");
// assert next.value instanceof Long;
// EdgeLabel el = (EdgeLabel)vertex.getTypeManager().getEdgeLabel(rt.name());
// log.debug("Retrieved ege label {} for name \"{}\"", el, rt.name());
// frel.setProperty(el, new FaunusVertex(configuration,(Long)next.value));
}
}
}
}

// // Iterate over and copy the relation's metaproperties
// if (relation.hasProperties()) {
// // Load relation properties
// for (final LongObjectCursor<Object> next : relation) {
// assert next.value != null;
// RelationType rt = typeManager.getExistingRelationType(next.key);
// if (rt.isPropertyKey()) {
// PropertyKey pkey = (PropertyKey)vertex.getTypeManager().getPropertyKey(rt.name());
// log.debug("Retrieved key {} for name \"{}\"", pkey, rt.name());
// frel.property(pkey.label(), next.value);
// } else {
// assert next.value instanceof Long;
// EdgeLabel el = (EdgeLabel)vertex.getTypeManager().getEdgeLabel(rt.name());
// log.debug("Retrieved ege label {} for name \"{}\"", el, rt.name());
// frel.setProperty(el, new FaunusVertex(configuration,(Long)next.value));
// }
// }
// for (JanusGraphRelation rel : frel.query().queryAll().relations())
// ((FaunusRelation)rel).setLifeCycle(ElementLifeCycle.Loaded);
// }
// frel.setLifeCycle(ElementLifeCycle.Loaded);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
//vertex.setLifeCycle(ElementLifeCycle.Loaded);

/*Since we are filtering out system relation types, we might end up with vertices that have no incident relations.
This is especially true for schema vertices. Those are filtered out. */
if (!foundVertexState) {
log.trace("Vertex {} has unknown lifecycle state", vertexId);
return null;
} else if (!tv.edges(Direction.BOTH).hasNext() && !tv.properties().hasNext()) {
log.trace("Vertex {} has no relations", vertexId);
return null;
}
return tv;
}