Uploaded image for project: 'Drools'
  1. Drools
  2. DROOLS-25

Cannot unmarshall large session file: InvalidProtocolBufferException

    XMLWordPrintable

Details

    • Bug
    • Resolution: Done
    • Major
    • 5.5.1.Final, 6.0.0.Alpha9
    • 5.5.0.Final
    • None
    • None
    • Hide

      Run the followoing java program. Tip: -Xmx4G

      MarshallUnmarshall.java
      package org.drools_25;
      
      import java.io.*;
      
      import org.drools.*;
      import org.drools.builder.*;
      import org.drools.io.*;
      import org.drools.marshalling.*;
      import org.drools.runtime.StatefulKnowledgeSession;
      
      public class MarshallUnmarshall {
      
          static String rules = 
                  "package org.drools_25 " + 
                  "" + 
                  "rule Populate_many " + 
                  "" + 
                  "when " + 
                  "   o : Integer(); " + 
                  "then " +
                  "   for(int i=0; i<5000000; i++) {" + 
                  "       insert(\"Fact: \" + i);" + 
                  "   }" + 
                  "end";
      
          public static void main(String[] args) throws Exception {
      
              StatefulKnowledgeSession kb = largeKb();
      
              File f = marshallKb(kb);
      
              kb.dispose();
      
              // Expect exception here
              unmarshallKb(f);
          }
      
          private static void unmarshallKb(File f) throws Exception {
      
              System.out.println("Reading from " + f.getName());
      
              FileInputStream in = new FileInputStream(f);
      
              Marshaller marshaller = MarshallerFactory
                      .newMarshaller(KnowledgeBaseFactory.newKnowledgeBase());
      
              marshaller.unmarshall(in);
          }
      
          private static File marshallKb(StatefulKnowledgeSession kb)
                  throws Exception {
      
              File kbfile = new File("out.bin");
      
              System.out.println("Writing to " + kbfile.getName()
                      + ". This will take a while...");
      
              FileOutputStream out = new FileOutputStream(kbfile);
      
              Marshaller marshaller = MarshallerFactory.newMarshaller(kb.getKnowledgeBase());
              
              marshaller.marshall(out, kb);
      
              return kbfile;
          }
      
          private static StatefulKnowledgeSession largeKb() {
      
              Resource myResource = ResourceFactory
                      .newReaderResource(new StringReader(rules));
      
              KnowledgeBuilder kbuilder = KnowledgeBuilderFactory
                      .newKnowledgeBuilder();
              kbuilder.add(myResource, ResourceType.DRL);
      
              KnowledgeBase kbase = KnowledgeBaseFactory.newKnowledgeBase();
              kbase.addKnowledgePackages(kbuilder.getKnowledgePackages());
      
              StatefulKnowledgeSession session = kbase.newStatefulKnowledgeSession();
              session.insert(new Integer(1));
      
              System.out.println("Filling up knowledge base.");
      
              session.fireAllRules();
      
              System.out.println("Done. Factcount = " + session.getFactCount());
      
              return session;
          }
      }
      
      Show
      Run the followoing java program. Tip: -Xmx4G MarshallUnmarshall.java package org.drools_25; import java.io.*; import org.drools.*; import org.drools.builder.*; import org.drools.io.*; import org.drools.marshalling.*; import org.drools.runtime.StatefulKnowledgeSession; public class MarshallUnmarshall { static String rules = " package org.drools_25 " + "" + "rule Populate_many " + "" + "when " + " o : Integer (); " + "then " + " for ( int i=0; i<5000000; i++) {" + " insert(\" Fact: \ " + i);" + " }" + "end" ; public static void main( String [] args) throws Exception { StatefulKnowledgeSession kb = largeKb(); File f = marshallKb(kb); kb.dispose(); // Expect exception here unmarshallKb(f); } private static void unmarshallKb(File f) throws Exception { System .out.println( "Reading from " + f.getName()); FileInputStream in = new FileInputStream(f); Marshaller marshaller = MarshallerFactory .newMarshaller(KnowledgeBaseFactory.newKnowledgeBase()); marshaller.unmarshall(in); } private static File marshallKb(StatefulKnowledgeSession kb) throws Exception { File kbfile = new File( "out.bin" ); System .out.println( "Writing to " + kbfile.getName() + ". This will take a while ..." ); FileOutputStream out = new FileOutputStream(kbfile); Marshaller marshaller = MarshallerFactory.newMarshaller(kb.getKnowledgeBase()); marshaller.marshall(out, kb); return kbfile; } private static StatefulKnowledgeSession largeKb() { Resource myResource = ResourceFactory .newReaderResource( new StringReader(rules)); KnowledgeBuilder kbuilder = KnowledgeBuilderFactory .newKnowledgeBuilder(); kbuilder.add(myResource, ResourceType.DRL); KnowledgeBase kbase = KnowledgeBaseFactory.newKnowledgeBase(); kbase.addKnowledgePackages(kbuilder.getKnowledgePackages()); StatefulKnowledgeSession session = kbase.newStatefulKnowledgeSession(); session.insert( new Integer (1)); System .out.println( "Filling up knowledge base." ); session.fireAllRules(); System .out.println( "Done. Factcount = " + session.getFactCount()); return session; } }

    Description

      When trying to unmarshall a large knowledge session from a file stream, an exception is thrown.

      It seems that there is a default message size limit in Protobuf (used by Drools marshalling). Apparently, what happens is that the marshalled "Header" message exceeds this size, which causes Protobuf to throw this exception.

      I suspect that the Header field "payload" is the culprit, since it seems to contain the rest of the knowledge base in byte array form.

      Here's the stack trace:

      Stacktrace
      com.google.protobuf.InvalidProtocolBufferException: Protocol message was too large. May be malicious. Use CodedInputStream.setSizeLimit() to increase the size limit.
          at com.google.protobuf.InvalidProtocolBufferException.sizeLimitExceeded:89
          at com.google.protobuf.CodedInputStream.refillBuffer:720
          at com.google.protobuf.CodedInputStream.isAtEnd:666
          at com.google.protobuf.CodedInputStream.readTag:99
          at org.drools.marshalling.impl.ProtobufMessages$Header$Builder.mergeFrom:967
          at org.drools.marshalling.impl.ProtobufMessages$Header$Builder.mergeFrom:773
          at com.google.protobuf.AbstractMessageLite$Builder.mergeFrom:212
          at com.google.protobuf.AbstractMessage$Builder.mergeFrom:746
          at org.drools.marshalling.impl.ProtobufMessages$Header.parseFrom:724
          at org.drools.marshalling.impl.PersisterHelper.readFromStreamWithHeader:234
          at org.drools.marshalling.impl.ProtobufInputMarshaller.loadAndParseSession:217
          at org.drools.marshalling.impl.ProtobufInputMarshaller.readSession:107
          at org.drools.marshalling.impl.ProtobufMarshaller.unmarshall:143
      

      Suggested solutions:

      #1
      You can set the size limit of the Protobuf CodedInputStream, but there doesn't seem to be any way of providing this parameter to the Drools unmarshalling API.

      #2
      When you unmarshall from a byte array, Protobuf does not enforce any message size limit. I have successfully implemented a patch in drools-core that preloads the file into memory and unmarshalls from a byte array. Probably the least desirable fix.

      Attachments

        Activity

          People

            etirelli@redhat.com Edson Tirelli
            magnusv_jira Magnus Vojbacke (Inactive)
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: