<security-role>
      <role-name>administrator</role-name>
</security-role>

<login-config>
      <auth-method>DIGEST</auth-method>
      <realm-name>WildFly8DigestRealm</realm-name>
</login-config>

<jboss-web>
    <security-domain>java:/jaas/my_secure_domain</security-domain>
</jboss-web>

<security-domain name="my_secure_domain" cache-type="default">
     <authentication>
          <login-module code="Database" flag="required">
               <module-option name="dsJndiName" value="java:jboss/datasources/MySqlDS"/>
               <module-option name="principalsQuery" value="select password from credential where uid=?"/>
               <module-option name="rolesQuery" value="select urole, 'Roles' from credential where uid=?"/>
               <module-option name="hashAlgorithm" value="MD5"/>
               <module-option name="hashEncoding" value="RFC2617"/>
               <module-option name="hashUserPassword" value="false"/>
               <module-option name="hashStorePassword" value="true"/>
               <module-option name="passwordIsA1Hash" value="true"/>
               <module-option name="digestCallback" value="org.jboss.security.auth.callback.DigestCallbackHandler"/>
               <module-option name="storeDigestCallback" value="org.jboss.security.auth.callback.RFC2617Digest"/>
          </login-module>
     </authentication>
</security-domain>

package com.aaa.encrypt;

import org.jboss.crypto.CryptoUtil;

public class EncryptPassword {

    public static void main(String[] args) {
        // TODO Auto-generated method stub
        String userName="admin"; 
        String realmName="WildFly8DigestRealm"; 
        String password="passwd123"; 

        String clearTextPassword=userName+":"+realmName+":"+password;  
        String hashedPassword=CryptoUtil.createPasswordHash("MD5", "RFC2617", null, null, clearTextPassword); 
        System.out.println("clearTextPassword: "+clearTextPassword); 
        System.out.println("hashedPassword: "+hashedPassword); 
    }
}

2014-07-18 21:37:45,246 TRACE [org.jboss.security] (default task-3) PBOX000236: Begin initialize method
2014-07-18 21:37:45,246 DEBUG [org.jboss.security] (default task-3) PBOX000281: Password hashing activated, algorithm: MD5, encoding: RFC2617, charset: null, callback: org.jboss.security.auth.callback.DigestCallbackHandler, storeCallBack: org.jboss.security.auth.callback.RFC2617Digest
2014-07-18 21:37:45,247 TRACE [org.jboss.security] (default task-3) PBOX000262: Module options [dsJndiName: java:jboss/datasources/MySqlDS, principalsQuery: select password from credential where uid=?, rolesQuery: select urole, 'Roles' from credential where uid=?, suspendResume: true]
2014-07-18 21:37:45,247 TRACE [org.jboss.security] (default task-3) PBOX000240: Begin login method
2014-07-18 21:37:45,249 TRACE [org.jboss.security] (default task-3) PBOX000263: Executing query select password from credential where uid=? with username admin
2014-07-18 21:37:45,251 TRACE [org.jboss.security] (default task-3) PBOX000284: Created DigestCallback org.jboss.security.auth.callback.RFC2617Digest
2014-07-18 21:37:45,252 TRACE [org.jboss.security] (default task-3) PBOX000244: Begin abort method
2014-07-18 21:37:45,252 DEBUG [org.jboss.security] (default task-3) PBOX000206: Login failure: javax.security.auth.login.LoginException: PBOX000055: Failed to invoke CallbackHandler
 at org.jboss.security.auth.spi.UsernamePasswordLoginModule.createPasswordHash(UsernamePasswordLoginModule.java:444) [picketbox-4.0.21.Beta1.jar:4.0.21.Beta1]
 at org.jboss.security.auth.spi.UsernamePasswordLoginModule.login(UsernamePasswordLoginModule.java:280) [picketbox-4.0.21.Beta1.jar:4.0.21.Beta1]
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [rt.jar:1.7.0_60]
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) [rt.jar:1.7.0_60]
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [rt.jar:1.7.0_60]
 at java.lang.reflect.Method.invoke(Method.java:606) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext.invoke(LoginContext.java:762) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext.access$000(LoginContext.java:203) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext$4.run(LoginContext.java:690) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext$4.run(LoginContext.java:688) [rt.jar:1.7.0_60]
 at java.security.AccessController.doPrivileged(Native Method) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:687) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext.login(LoginContext.java:595) [rt.jar:1.7.0_60]
 at org.jboss.security.authentication.JBossCachedAuthenticationManager.defaultLogin(JBossCachedAuthenticationManager.java:408) [picketbox-infinispan-4.0.21.Beta1.jar:4.0.21.Beta1]
 at org.jboss.security.authentication.JBossCachedAuthenticationManager.proceedWithJaasLogin(JBossCachedAuthenticationManager.java:345) [picketbox-infinispan-4.0.21.Beta1.jar:4.0.21.Beta1]
 at org.jboss.security.authentication.JBossCachedAuthenticationManager.authenticate(JBossCachedAuthenticationManager.java:333) [picketbox-infinispan-4.0.21.Beta1.jar:4.0.21.Beta1]
 at org.jboss.security.authentication.JBossCachedAuthenticationManager.isValid(JBossCachedAuthenticationManager.java:146) [picketbox-infinispan-4.0.21.Beta1.jar:4.0.21.Beta1]
 at org.wildfly.extension.undertow.security.JAASIdentityManagerImpl.verifyCredential(JAASIdentityManagerImpl.java:111)
 at org.wildfly.extension.undertow.security.JAASIdentityManagerImpl.verify(JAASIdentityManagerImpl.java:77)
 at io.undertow.security.impl.DigestAuthenticationMechanism.handleDigestHeader(DigestAuthenticationMechanism.java:265) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.DigestAuthenticationMechanism.authenticate(DigestAuthenticationMechanism.java:149) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.SecurityContextImpl$AuthAttempter.transition(SecurityContextImpl.java:281) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.SecurityContextImpl$AuthAttempter.transition(SecurityContextImpl.java:298) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.SecurityContextImpl$AuthAttempter.access$100(SecurityContextImpl.java:268) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.SecurityContextImpl.attemptAuthentication(SecurityContextImpl.java:131) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.SecurityContextImpl.authTransition(SecurityContextImpl.java:106) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.impl.SecurityContextImpl.authenticate(SecurityContextImpl.java:99) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.security.ServletAuthenticationCallHandler.handleRequest(ServletAuthenticationCallHandler.java:54) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.server.handlers.DisableCacheHandler.handleRequest(DisableCacheHandler.java:27) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:25) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.handlers.AuthenticationConstraintHandler.handleRequest(AuthenticationConstraintHandler.java:51) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.handlers.AbstractConfidentialityHandler.handleRequest(AbstractConfidentialityHandler.java:45) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.security.ServletConfidentialityConstraintHandler.handleRequest(ServletConfidentialityConstraintHandler.java:61) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.security.ServletSecurityConstraintHandler.handleRequest(ServletSecurityConstraintHandler.java:56) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.handlers.AuthenticationMechanismsHandler.handleRequest(AuthenticationMechanismsHandler.java:58) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.security.CachedAuthenticatedSessionHandler.handleRequest(CachedAuthenticatedSessionHandler.java:70) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.security.handlers.SecurityInitialHandler.handleRequest(SecurityInitialHandler.java:76) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:25) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at org.wildfly.extension.undertow.security.jacc.JACCContextIdHandler.handleRequest(JACCContextIdHandler.java:61)
 at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:25) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.server.handlers.PredicateHandler.handleRequest(PredicateHandler.java:25) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.ServletInitialHandler.handleFirstRequest(ServletInitialHandler.java:240) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.ServletInitialHandler.dispatchRequest(ServletInitialHandler.java:227) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.ServletInitialHandler.access$000(ServletInitialHandler.java:73) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.servlet.handlers.ServletInitialHandler$1.handleRequest(ServletInitialHandler.java:146) [undertow-servlet-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.server.Connectors.executeRootHandler(Connectors.java:177) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at io.undertow.server.HttpServerExchange$1.run(HttpServerExchange.java:727) [undertow-core-1.0.15.Final.jar:1.0.15.Final]
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) [rt.jar:1.7.0_60]
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) [rt.jar:1.7.0_60]
 at java.lang.Thread.run(Thread.java:745) [rt.jar:1.7.0_60]

Caused by: javax.security.auth.callback.UnsupportedCallbackException
 at org.jboss.security.auth.callback.JBossCallbackHandler.handleCallBack(JBossCallbackHandler.java:138) [picketbox-4.0.21.Beta1.jar:4.0.21.Beta1]
 at org.jboss.security.auth.callback.JBossCallbackHandler.handle(JBossCallbackHandler.java:87) [picketbox-4.0.21.Beta1.jar:4.0.21.Beta1]
 at javax.security.auth.login.LoginContext$SecureCallbackHandler$1.run(LoginContext.java:947) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext$SecureCallbackHandler$1.run(LoginContext.java:944) [rt.jar:1.7.0_60]
 at java.security.AccessController.doPrivileged(Native Method) [rt.jar:1.7.0_60]
 at javax.security.auth.login.LoginContext$SecureCallbackHandler.handle(LoginContext.java:943) [rt.jar:1.7.0_60]
 at org.jboss.security.auth.spi.UsernamePasswordLoginModule.createPasswordHash(UsernamePasswordLoginModule.java:434) [picketbox-4.0.21.Beta1.jar:4.0.21.Beta1]
 ... 49 more