Re: reindex job is very slow on ElasticSearch and BigTable


vamsi.lingala@...
 

I am getting below errors while doing reindex and it stops the job.

Pls help me with this.

09:09:06.295 [Thread-12] ERROR o.j.d.k.scan.StandardScannerExecutor Unexpected error processing data: {}
org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:285) ~[janusgraph-core-0.5.1.jar:na]
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201) ~[janusgraph-core-0.5.1.jar:na]
	... 2 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-4 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893) ~[janusgraph-es-0.5.1.jar:na]
	... 4 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-4 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_275]
09:09:06.438 [Thread-13] ERROR o.j.d.k.scan.StandardScannerExecutor Unexpected error processing data: {}
org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:285) ~[janusgraph-core-0.5.1.jar:na]
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201) ~[janusgraph-core-0.5.1.jar:na]
	... 2 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-2 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893) ~[janusgraph-es-0.5.1.jar:na]
	... 4 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-2 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_275]
09:09:07.290 [Thread-10] ERROR o.j.d.k.scan.StandardScannerExecutor Unexpected error processing data: {}
org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:285) ~[janusgraph-core-0.5.1.jar:na]
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201) ~[janusgraph-core-0.5.1.jar:na]
	... 2 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-0 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893) ~[janusgraph-es-0.5.1.jar:na]
	... 4 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-0 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_275]
09:09:07.325 [Thread-14] ERROR o.j.d.k.scan.StandardScannerExecutor Unexpected error processing data: {}
org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:285) ~[janusgraph-core-0.5.1.jar:na]
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201) ~[janusgraph-core-0.5.1.jar:na]
	... 2 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-1 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893) ~[janusgraph-es-0.5.1.jar:na]
	... 4 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-1 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_275]
09:09:08.323 [Thread-11] ERROR o.j.d.k.scan.StandardScannerExecutor Unexpected error processing data: {}
org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:285) ~[janusgraph-core-0.5.1.jar:na]
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128) ~[janusgraph-core-0.5.1.jar:na]
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201) ~[janusgraph-core-0.5.1.jar:na]
	... 2 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-3 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-7.6.1.jar:7.6.1]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397) ~[janusgraph-es-0.5.1.jar:na]
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893) ~[janusgraph-es-0.5.1.jar:na]
	... 4 common frames omitted
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-3 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39) ~[httpasyncclient-4.1.4.jar:4.1.4]
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591) ~[httpcore-nio-4.4.12.jar:4.4.12]
	at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_275]
Exception in thread "Thread-12" org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206)
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118)
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:305)
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895)
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128)
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201)
	... 2 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-5 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893)
	... 4 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-5 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
	at java.lang.Thread.run(Thread.java:748)
Exception in thread "Thread-13" org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206)
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118)
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:305)
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895)
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128)
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201)
	... 2 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-6 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893)
	... 4 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-6 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
	at java.lang.Thread.run(Thread.java:748)
Exception in thread "Thread-10" org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206)
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118)
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:305)
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895)
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128)
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201)
	... 2 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-7 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893)
	... 4 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-7 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
	at java.lang.Thread.run(Thread.java:748)
Exception in thread "Thread-14" org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206)
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118)
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:305)
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895)
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128)
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201)
	... 2 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-8 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893)
	... 4 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-8 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
	at java.lang.Thread.run(Thread.java:748)
Exception in thread "Thread-11" org.janusgraph.core.JanusGraphException: Unknown exception while executing index operation
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:206)
	at org.janusgraph.graphdb.olap.VertexJobConverter.workerIterationEnd(VertexJobConverter.java:118)
	at org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScannerExecutor$Processor.run(StandardScannerExecutor.java:305)
Caused by: org.janusgraph.diskstorage.PermanentBackendException: Unknown exception while executing index operation
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.convert(ElasticSearchIndex.java:447)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:895)
	at org.janusgraph.diskstorage.indexing.IndexTransaction.restore(IndexTransaction.java:128)
	at org.janusgraph.graphdb.olap.job.IndexRepairJob.workerIterationEnd(IndexRepairJob.java:201)
	... 2 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-9 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:808)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:248)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:483)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.performRequest(RestElasticSearchClient.java:474)
	at org.janusgraph.diskstorage.es.rest.RestElasticSearchClient.bulkRequest(RestElasticSearchClient.java:397)
	at org.janusgraph.diskstorage.es.ElasticSearchIndex.restore(ElasticSearchIndex.java:893)
	... 4 more
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-9 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
	at java.lang.Thread.run(Thread.java:748)
09:18:46.489 [bigtable-rpc-retry-3] WARN  c.g.cloud.bigtable.grpc.io.Watchdog Found 3 stale streams and cancelled them
09:19:57.643 [Credentials-Refresh-8] INFO  c.g.c.b.g.io.OAuthCredentialsCache Refreshing the OAuth token

Join janusgraph-users@lists.lfaidata.foundation to automatically receive all group messages.