Hi guys,
During async process instance migration of a process definition with 150K instances a exception occures reccuring.
The batch runs in a homogenous cluster with 4 nodes.
JobExecutor is NOT deployment aware (because it is a homogenous cluster so not necessary).
The plain camunda process instance migration functionality is used.
The batch has many failed jobs because of the exception (see below).
Maybe a bug in migration functionality?
Best regards,
Markus
2017-03-19 18:29:23,214 [pool-2-thread-1] WARN org.camunda.bpm.engine.jobexecutor - ENGINE-14006 Exception while executing job 68ef86b3-0cc8-11e7-8dcc-005056a16247:
org.camunda.bpm.engine.ProcessEngineException: ENGINE-03004 Exception while executing Database Operation 'DELETE ByteArrayEntity[7f3bf87b-0bcc-11e7-9d50-005056a17feb]' with message '
### Error updating database. Cause: java.sql.SQLException: ORA-00060: deadlock detected while waiting for resource
### The error may involve org.camunda.bpm.engine.impl.persistence.entity.VariableInstanceEntity.deleteByteArray-Inline
### The error occurred while setting parameters
### SQL: delete from BUSINESSPARTNER.ACT_GE_BYTEARRAY where ID_ = ? and REV_ = ?
### Cause: java.sql.SQLException: ORA-00060: deadlock detected while waiting for resource
'. Flush summary:
[
INSERT ByteArrayEntity[8e8ca0b2-0cc9-11e7-8cc9-005056a17feb]
INSERT ByteArrayEntity[8e921ef3-0cc9-11e7-8cc9-005056a17feb]
INSERT ByteArrayEntity[8e929424-0cc9-11e7-8cc9-005056a17feb]
DELETE MessageEntity[68ef86b3-0cc8-11e7-8dcc-005056a16247]
UPDATE TimerEntity[7f3dcd46-0bcc-11e7-9d50-005056a17feb]
UPDATE TimerEntity[7f4101a2-0bcc-11e7-9d50-005056a17feb]
DELETE ByteArrayEntity[68ef86b2-0cc8-11e7-8dcc-005056a16247]
DELETE ByteArrayEntity[7f3bf870-0bcc-11e7-9d50-005056a17feb]
DELETE ByteArrayEntity[7f3bf87b-0bcc-11e7-9d50-005056a17feb]
DELETE ByteArrayEntity[7f3bf87f-0bcc-11e7-9d50-005056a17feb]
DELETE_BULK deleteByteArrayNoRevisionCheck 68ef86b2-0cc8-11e7-8dcc-005056a16247
UPDATE TaskEntity[7f3dcd47-0bcc-11e7-9d50-005056a17feb]
UPDATE TaskEntity[7f4101a3-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f3d5814-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f3bf880-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f3bf876-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f408c70-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f3fa20c-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f3bf877-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[7f3bf871-0bcc-11e7-9d50-005056a17feb]
UPDATE ExecutionEntity[13fca326-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricActivityInstanceEventEntity[7f3bf872-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricActivityInstanceEventEntity[SubProcess_108a19j:7f3bf881-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricActivityInstanceEventEntity[SubProcess_108a19j:7f3fa20d-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricActivityInstanceEventEntity[fulfillRequirenmentUserTask:7f3d5815-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricActivityInstanceEventEntity[fulfillRequirenmentUserTask:7f408c71-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricProcessInstanceEventEntity[13fca326-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricTaskInstanceEventEntity[7f3dcd47-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricTaskInstanceEventEntity[7f4101a3-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca327-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca328-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca329-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca32a-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca32b-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca32c-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[13fca32d-0bcb-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf86f-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf873-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf874-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf875-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf878-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf87a-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf87c-0bcc-11e7-9d50-005056a17feb]
UPDATE HistoricVariableInstanceEntity[7f3bf87e-0bcc-11e7-9d50-005056a17feb]
]
at org.camunda.bpm.engine.impl.db.EnginePersistenceLogger.flushDbOperationException(EnginePersistenceLogger.java:115)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flushDbOperationManager(DbEntityManager.java:292)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flush(DbEntityManager.java:278)
at org.camunda.bpm.engine.impl.interceptor.CommandContext.flushSessions(CommandContext.java:247)
at org.camunda.bpm.engine.impl.interceptor.CommandContext.close(CommandContext.java:176)
at org.camunda.bpm.engine.impl.interceptor.CommandContextInterceptor.execute(CommandContextInterceptor.java:113)
at org.camunda.bpm.engine.spring.SpringTransactionInterceptor$1.doInTransaction(SpringTransactionInterceptor.java:42)
at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133)
at org.camunda.bpm.engine.spring.SpringTransactionInterceptor.execute(SpringTransactionInterceptor.java:40)
at org.camunda.bpm.engine.impl.interceptor.ProcessApplicationContextInterceptor.execute(ProcessApplicationContextInterceptor.java:66)
at org.camunda.bpm.engine.impl.interceptor.LogInterceptor.execute(LogInterceptor.java:30)
at org.camunda.bpm.engine.impl.jobexecutor.ExecuteJobHelper.executeJob(ExecuteJobHelper.java:35)
at org.camunda.bpm.engine.impl.jobexecutor.ExecuteJobHelper.executeJob(ExecuteJobHelper.java:28)
at org.camunda.bpm.engine.impl.jobexecutor.ExecuteJobsRunnable.executeJob(ExecuteJobsRunnable.java:82)
at org.camunda.bpm.engine.impl.jobexecutor.ExecuteJobsRunnable.run(ExecuteJobsRunnable.java:56)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.ibatis.exceptions.PersistenceException:
### Error updating database. Cause: java.sql.SQLException: ORA-00060: deadlock detected while waiting for resource
### The error may involve org.camunda.bpm.engine.impl.persistence.entity.VariableInstanceEntity.deleteByteArray-Inline
### The error occurred while setting parameters
### SQL: delete from BUSINESSPARTNER.ACT_GE_BYTEARRAY where ID_ = ? and REV_ = ?
### Cause: java.sql.SQLException: ORA-00060: deadlock detected while waiting for resource
at org.apache.ibatis.exceptions.ExceptionFactory.wrapException(ExceptionFactory.java:26)
at org.apache.ibatis.session.defaults.DefaultSqlSession.update(DefaultSqlSession.java:154)
at org.apache.ibatis.session.defaults.DefaultSqlSession.delete(DefaultSqlSession.java:165)
at org.camunda.bpm.engine.impl.db.sql.DbSqlSession.executeDelete(DbSqlSession.java:187)
at org.camunda.bpm.engine.impl.db.sql.DbSqlSession.deleteEntity(DbSqlSession.java:172)
at org.camunda.bpm.engine.impl.db.AbstractPersistenceSession.executeDbOperation(AbstractPersistenceSession.java:45)
at org.camunda.bpm.engine.impl.db.entitymanager.DbEntityManager.flushDbOperationManager(DbEntityManager.java:289)
... 16 common frames omitted
Caused by: java.sql.SQLException: ORA-00060: deadlock detected while waiting for resource
at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:439)
at oracle.jdbc.driver.T4CTTIoer.processError(T4CTTIoer.java:395)
at oracle.jdbc.driver.T4C8Oall.processError(T4C8Oall.java:802)
at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:436)
at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:186)
at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:521)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:205)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:1008)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1307)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3449)
at oracle.jdbc.driver.OraclePreparedStatement.execute(OraclePreparedStatement.java:3550)
at oracle.jdbc.driver.OraclePreparedStatementWrapper.execute(OraclePreparedStatementWrapper.java:1374)
at org.apache.ibatis.executor.statement.PreparedStatementHandler.update(PreparedStatementHandler.java:44)
at org.apache.ibatis.executor.statement.RoutingStatementHandler.update(RoutingStatementHandler.java:69)
at org.apache.ibatis.executor.SimpleExecutor.doUpdate(SimpleExecutor.java:48)
at org.apache.ibatis.executor.BaseExecutor.update(BaseExecutor.java:105)
at org.apache.ibatis.executor.CachingExecutor.update(CachingExecutor.java:71)
at org.apache.ibatis.session.defaults.DefaultSqlSession.update(DefaultSqlSession.java:152)