Uploaded image for project: 'Debezium'
  1. Debezium
  2. DBZ-4605

JSON data corrupted in update events

    XMLWordPrintable

Details

    • Hide

       

      // create table
      CREATE TABLE IF NOT EXISTS deals (
          id int(11) unsigned NOT NULL AUTO_INCREMENT,
          company_id int(11) unsigned DEFAULT NULL,
          title varchar(255) DEFAULT NULL,
          custom_fields json DEFAULT NULL,
          PRIMARY KEY (id),
          KEY idx_company_id (company_id)
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT; 

       

       

       

      // insert one row with json data
      INSERT INTO deals (title, custom_fields) VALUES ('test', "{\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",\"1f3a2ea5bc1f60258df20521bee9ac636df69a3a\":{\"currency\":\"USD\"},\"4f4d99a438f334d7dbf83a1816015b361b848b3b\":{\"currency\":\"USD\"},\"9021162291be72f5a8025480f44bf44d5d81d07c\":\"test_field_for_remove_fields_behaviour_3_will_be_removed\",\"9b0ed11532efea688fdf12b28f142b9eb08a80c5\":{\"currency\":\"USD\"},\"e65ad0762c259b05b4866f7249eabecabadbe577\":\"test_field_for_remove_fields_behaviour_1_updated\",\"ff2c07edcaa3e987c23fb5cc4fe860bb52becf00\":{\"currency\":\"USD\"}}");
      

      // create event arrives correctly

       

       

       

      // now remove first json data property
      UPDATE deals SET custom_fields = JSON_REMOVE(custom_fields, '$."17fc9889474028063990914001f6854f6b8b5784"') 

       

      // corrupted update event arrives (in after part custom_field properties all are corrupted, yet the before state field names are correct!)

       

      // EVENT json with pretty print
      {
        "before": {
          "id": 1,
          "company_id": null,
          "title": "test",
          "custom_fields": "{\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",\"1f3a2ea5bc1f60258df20521bee9ac636df69a3a\":{\"currency\":\"USD\"},\"4f4d99a438f334d7dbf83a1816015b361b848b3b\":{\"currency\":\"USD\"},\"9021162291be72f5a8025480f44bf44d5d81d07c\":\"test_field_for_remove_fields_behaviour_3_will_be_removed\",\"9b0ed11532efea688fdf12b28f142b9eb08a80c5\":{\"currency\":\"USD\"},\"e65ad0762c259b05b4866f7249eabecabadbe577\":\"test_field_for_remove_fields_behaviour_1_updated\",\"ff2c07edcaa3e987c23fb5cc4fe860bb52becf00\":{\"currency\":\"USD\"}}"
        },
        "after": {
          "id": 1,
          "company_id": null,
          "title": "test",
          "custom_fields": "{\"\\u0001\\f�\\u0001\\u0000%\\u000217fc9889474028063990914001f6854f6\":{\"currency\":\"USD\"},\"b8b57841f3a2ea5bc1f60258df20521bee9ac636\":{\"currency\":\"USD\"},\"df69a3a4f4d99a438f334d7dbf83a1816015b361\":\"test_field_for_remove_fields_behaviour_3_will_be_removed\",\"b848b3b9021162291be72f5a8025480f44bf44d5\":{\"currency\":\"USD\"},\"d81d07c9b0ed11532efea688fdf12b28f142b9eb\":\"test_field_for_remove_fields_behaviour_1_updated\",\"08a80c5e65ad0762c259b05b4866f7249eabecab\":{\"currency\":\"USD\"}}"
        },
        "source": {
          "version": "1.8.0.Final",
          "connector": "mysql",
          "name": "dbserver1",
          "ts_ms": 1643050258000,
          "snapshot": "false",
          "db": "inventory",
          "sequence": null,
          "table": "deals",
          "server_id": 223344,
          "gtid": null,
          "file": "mysql-bin.000003",
          "pos": 1764,
          "row": 0,
          "thread": null,
          "query": null
        },
        "op": "u",
        "ts_ms": 1643050258643,
        "transaction": null
      }
      
      

       

       

       

      Show
        // create table CREATE TABLE IF NOT EXISTS deals (     id int (11) unsigned NOT NULL AUTO_INCREMENT,     company_id int (11) unsigned DEFAULT NULL,     title varchar(255) DEFAULT NULL,     custom_fields json DEFAULT NULL,     PRIMARY KEY (id),     KEY idx_company_id (company_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT;       // insert one row with json data INSERT INTO deals (title, custom_fields) VALUES ( 'test' , "{\" 17fc9889474028063990914001f6854f6b8b5784\ ":\" test_field_for_remove_fields_behaviour_2\ ",\" 1f3a2ea5bc1f60258df20521bee9ac636df69a3a\ ":{\" currency\ ":\" USD\ "},\" 4f4d99a438f334d7dbf83a1816015b361b848b3b\ ":{\" currency\ ":\" USD\ "},\" 9021162291be72f5a8025480f44bf44d5d81d07c\ ":\" test_field_for_remove_fields_behaviour_3_will_be_removed\ ",\" 9b0ed11532efea688fdf12b28f142b9eb08a80c5\ ":{\" currency\ ":\" USD\ "},\" e65ad0762c259b05b4866f7249eabecabadbe577\ ":\" test_field_for_remove_fields_behaviour_1_updated\ ",\" ff2c07edcaa3e987c23fb5cc4fe860bb52becf00\ ":{\" currency\ ":\" USD\ "}}" ); // create event arrives correctly       // now remove first json data property UPDATE deals SET custom_fields = JSON_REMOVE(custom_fields, '$. "17fc9889474028063990914001f6854f6b8b5784" ' )   // corrupted update event arrives (in after part custom_field properties all are corrupted, yet the before state field names are correct!)   // EVENT json with pretty print {   "before" : {     "id" : 1,     "company_id" : null ,     "title" : "test" ,     "custom_fields" : "{\" 17fc9889474028063990914001f6854f6b8b5784\ ":\" test_field_for_remove_fields_behaviour_2\ ",\" 1f3a2ea5bc1f60258df20521bee9ac636df69a3a\ ":{\" currency\ ":\" USD\ "},\" 4f4d99a438f334d7dbf83a1816015b361b848b3b\ ":{\" currency\ ":\" USD\ "},\" 9021162291be72f5a8025480f44bf44d5d81d07c\ ":\" test_field_for_remove_fields_behaviour_3_will_be_removed\ ",\" 9b0ed11532efea688fdf12b28f142b9eb08a80c5\ ":{\" currency\ ":\" USD\ "},\" e65ad0762c259b05b4866f7249eabecabadbe577\ ":\" test_field_for_remove_fields_behaviour_1_updated\ ",\" ff2c07edcaa3e987c23fb5cc4fe860bb52becf00\ ":{\" currency\ ":\" USD\ "}}"   },   "after" : {     "id" : 1,     "company_id" : null ,     "title" : "test" ,     "custom_fields" : "{\" \\u0001\\f�\\u0001\\u0000%\\u000217fc9889474028063990914001f6854f6\ ":{\" currency\ ":\" USD\ "},\" b8b57841f3a2ea5bc1f60258df20521bee9ac636\ ":{\" currency\ ":\" USD\ "},\" df69a3a4f4d99a438f334d7dbf83a1816015b361\ ":\" test_field_for_remove_fields_behaviour_3_will_be_removed\ ",\" b848b3b9021162291be72f5a8025480f44bf44d5\ ":{\" currency\ ":\" USD\ "},\" d81d07c9b0ed11532efea688fdf12b28f142b9eb\ ":\" test_field_for_remove_fields_behaviour_1_updated\ ",\" 08a80c5e65ad0762c259b05b4866f7249eabecab\ ":{\" currency\ ":\" USD\ "}}"   },   "source" : {     "version" : "1.8.0.Final" ,     "connector" : "mysql" ,     "name" : "dbserver1" ,     "ts_ms" : 1643050258000,     "snapshot" : " false " ,     "db" : "inventory" ,     "sequence" : null ,     "table" : "deals" ,     "server_id" : 223344,     "gtid" : null ,     "file" : "mysql-bin.000003" ,     "pos" : 1764,     "row" : 0,     "thread" : null ,     "query" : null   },   "op" : "u" ,   "ts_ms" : 1643050258643,   "transaction" : null }      

    Description

      Seems after certain JSON command and object data the debezium event in "after" state fields property names get corrupted. We've encountered this in production and it reproduces always. I also tested it with your latest 1.8Final tutorial docker-compose file... Very unexpected how it happens

      Attachments

        Activity

          People

            jpechane Jiri Pechanec
            pimpelsang Eero Koplimets (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: