Uploaded image for project: 'Debezium'
  1. Debezium
  2. DBZ-3377

Connector crashing after running for some time

    XMLWordPrintable

Details

    Description

      Hello!

      I have the following connector

      {
          "name": "kafka-rds-dev-003",
          "config": {
              "connector.class": "io.debezium.connector.oracle.OracleConnector",
              "tasks.max": "1",
              "database.server.name": "databasename",
              "database.hostname": "host",
              "database.port": "1521",
              "database.user": "username",
              "database.password": "password",
              "database.dbname": "SDIGCDEV",
              "database.history.kafka.bootstrap.servers": "http://kafka-confluent-cp-kafka.kafka-rds.svc.cluster.local:9092",
              "database.history.kafka.topic": "schema-kafka-rds-dev",
              "key.converter": "io.confluent.connect.avro.AvroConverter",
              "key.converter.schema.registry.url": "http://kafka-confluent-cp-schema-registry.kafka-rds.svc.cluster.local:8081",
              "value.converter": "io.confluent.connect.avro.AvroConverter",
              "value.converter.schema.registry.url": "http://kafka-confluent-cp-schema-registry.kafka-rds.svc.cluster.local:8081",
              "errors.log.enable": "true",
              "database.connection.adapter": "logminer",
              "log.mining.archive.log.hours": 24,
              "log.mining.strategy": "online_catalog",
              "snapshot.mode": "schema_only",
      
              "schema.include.list": "SIEBEL",
              "table.include.list": "SIEBEL.S_ASSET,SIEBEL.S_ASSET_XA,SIEBEL.S_ORG_EXT,SIEBEL.S_PROD_INT",
              "column.include.list": "SIEBEL.S_ASSET.ROW_ID,SIEBEL.S_ASSET.BILL_ACCNT_ID,SIEBEL.S_ASSET.STATUS_CD,SIEBEL.S_ASSET.PROD_ID,SIEBEL.S_ASSET.ASSET_NUM,SIEBEL.S_ASSET.START_DT,SIEBEL.S_ASSET.END_DT,SIEBEL.S_ASSET_XA.ROW_ID,SIEBEL.S_ASSET_XA.CHAR_VAL,SIEBEL.S_ASSET_XA.ASSET_ID,SIEBEL.S_ASSET_XA.ATTR_NAME,SIEBEL.S_ORG_EXT.ROW_ID,SIEBEL.S_ORG_EXT.OU_SUM,SIEBEL.S_ORG_EXT.OU_NUM,SIEBEL.S_ORG_EXT.OU_TYPE_CD,SIEBEL.S_ORG_EXT.X_DUNN_LEVEL,SIEBEL.S_ORG_EXT.CUST_STAT_CD,SIEBEL.S_PROD_INT.CATEGORY_CD,SIEBEL.S_PROD_INT.SUB_TYPE_CD,SIEBEL.S_PROD_INT.ROW_ID,SIEBEL.S_PROD_INT.NAME",
      
              "transforms": "unwrap",
      
              "transforms.unwrap.type": "io.debezium.transforms.ExtractNewRecordState",
              "transforms.unwrap.drop.tombstones": "false",
              "transforms.unwrap.delete.handling.mode": "rewrite"
      
          }
      }
      

      All works fine, but after 1 day, sometimes less than 1 day running I got the following error:

      {
          "name": "kafka-rds-dev-003",
          "connector": {
              "state": "RUNNING",
              "worker_id": "172.27.19.11:8083"
          },
          "tasks": [
              {
                  "id": 0,
                  "state": "FAILED",
                  "worker_id": "172.27.19.11:8083",
                  "trace": "org.apache.kafka.connect.errors.ConnectException: An exception occurred in the change event producer. This connector will be stopped.\n\tat io.debezium.pipeline.ErrorHandler.setProducerThrowable(ErrorHandler.java:42)\n\tat io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource.execute(LogMinerStreamingChangeEventSource.java:204)\n\tat io.debezium.pipeline.ChangeEventSourceCoordinator.streamEvents(ChangeEventSourceCoordinator.java:140)\n\tat io.debezium.pipeline.ChangeEventSourceCoordinator.lambda$start$0(ChangeEventSourceCoordinator.java:113)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)\n\tat java.base/java.lang.Thread.run(Thread.java:834)\nCaused by: java.sql.SQLException: ORA-00310: archived log contains sequence 3256986; sequence 3256982 required\nORA-00334: archived log: '/rdsdbdata/db/SDIGCDEV_A/onlinelog/o1_mf_2_d20tr0jl_.log'\n\n\tat oracle.jdbc.driver.T4CTTIoer11.processError(T4CTTIoer11.java:628)\n\tat oracle.jdbc.driver.T4CTTIoer11.processError(T4CTTIoer11.java:562)\n\tat oracle.jdbc.driver.T4C8Oall.processError(T4C8Oall.java:1145)\n\tat oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:726)\n\tat oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:291)\n\tat oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:492)\n\tat oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:148)\n\tat oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:1038)\n\tat oracle.jdbc.driver.OracleStatement.executeMaybeDescribe(OracleStatement.java:1097)\n\tat oracle.jdbc.driver.OracleStatement.executeSQLSelect(OracleStatement.java:1402)\n\tat oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1285)\n\tat oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3735)\n\tat oracle.jdbc.driver.OraclePreparedStatement.executeQuery(OraclePreparedStatement.java:3847)\n\tat oracle.jdbc.driver.OraclePreparedStatementWrapper.executeQuery(OraclePreparedStatementWrapper.java:1098)\n\tat io.debezium.connector.oracle.logminer.LogMinerStreamingChangeEventSource.execute(LogMinerStreamingChangeEventSource.java:180)\n\t... 7 more\nCaused by: Error : 310, Position : 0, Sql = SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME FROM V$LOGMNR_CONTENTS WHERE SCN > :1  AND SCN <= :2  AND ((OPERATION_CODE IN (5,34) AND USERNAME NOT IN ('SYS','SYSTEM','LOGMINER')) OR (OPERATION_CODE IN (7,36)) OR (OPERATION_CODE IN (1,2,3) AND TABLE_NAME != 'LOG_MINING_FLUSH' AND SEG_OWNER NOT IN ('APPQOSSYS','AUDSYS','CTXSYS','DVSYS','DBSFWUSER','DBSNMP','GSMADMIN_INTERNAL','LBACSYS','MDSYS','OJVMSYS','OLAPSYS','ORDDATA','ORDSYS','OUTLN','SYS','SYSTEM','WMSYS','XDB') AND (REGEXP_LIKE(SEG_OWNER,'^SIEBEL$','i')) AND (REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_ASSET$','i') OR REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_ASSET_XA$','i') OR REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_ORG_EXT$','i') OR REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_PROD_INT$','i')) )), OriginalSql = SELECT SCN, SQL_REDO, OPERATION_CODE, TIMESTAMP, XID, CSF, TABLE_NAME, SEG_OWNER, OPERATION, USERNAME FROM V$LOGMNR_CONTENTS WHERE SCN > ? AND SCN <= ? AND ((OPERATION_CODE IN (5,34) AND USERNAME NOT IN ('SYS','SYSTEM','LOGMINER')) OR (OPERATION_CODE IN (7,36)) OR (OPERATION_CODE IN (1,2,3) AND TABLE_NAME != 'LOG_MINING_FLUSH' AND SEG_OWNER NOT IN ('APPQOSSYS','AUDSYS','CTXSYS','DVSYS','DBSFWUSER','DBSNMP','GSMADMIN_INTERNAL','LBACSYS','MDSYS','OJVMSYS','OLAPSYS','ORDDATA','ORDSYS','OUTLN','SYS','SYSTEM','WMSYS','XDB') AND (REGEXP_LIKE(SEG_OWNER,'^SIEBEL$','i')) AND (REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_ASSET$','i') OR REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_ASSET_XA$','i') OR REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_ORG_EXT$','i') OR REGEXP_LIKE(SEG_OWNER || '.' || TABLE_NAME,'^SIEBEL.S_PROD_INT$','i')) )), Error Msg = ORA-00310: archived log contains sequence 3256986; sequence 3256982 required\nORA-00334: archived log: '/rdsdbdata/db/SDIGCDEV_A/onlinelog/o1_mf_2_d20tr0jl_.log'\n\n\tat oracle.jdbc.driver.T4CTTIoer11.processError(T4CTTIoer11.java:632)\n\t... 21 more\n"
              }
          ],
          "type": "source"
      }
      

      I don't know what is going on... can you guys help me?

      thanks!

      Attachments

        Activity

          People

            ccranfor@redhat.com Chris Cranford
            thiago-ananias thiago ananias silva (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: