Keycloak 17 example using JGroups JDBC_PING discovery protocol for Infinispan

cache-ispn-jdbc-ping.xml

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="urn:infinispan:config:11.0 http://www.infinispan.org/schemas/infinispan-config-11.0.xsd"
    xmlns="urn:infinispan:config:11.0">

  <!-- custom stack goes into the jgroups element -->
  <jgroups>
    <stack name="jdbc-ping-tcp" extends="tcp">
      <JDBC_PING connection_driver="org.postgresql.Driver"
                 connection_username="${env.KC_DB_USERNAME}" connection_password="${env.KC_DB_PASSWORD}"
                 connection_url="jdbc:postgresql://${env.KC_DB_URL_HOST}/${env.KC_DB_URL_DATABASE}"
                 initialize_sql="CREATE TABLE IF NOT EXISTS JGROUPSPING (own_addr varchar(200) NOT NULL, cluster_name varchar(200) NOT NULL, ping_data BYTEA, constraint PK_JGROUPSPING PRIMARY KEY (own_addr, cluster_name));"
                 info_writer_sleep_time="500"
                 remove_all_data_on_view_change="true"
                 stack.combine="REPLACE"
                 stack.position="MPING" />
    </stack>
  </jgroups>
  
  <cache-container name="keycloak">
    <!-- custom stack must be referenced by name in the stack attribute of the transport element -->
    <transport lock-timeout="60000" stack="jdbc-ping-tcp"/>
    <local-cache name="realms">
      <encoding>
        <key media-type="application/x-java-object"/>
        <value media-type="application/x-java-object"/>
      </encoding>
      <memory max-count="10000"/>
    </local-cache>
    <local-cache name="users">
      <encoding>
        <key media-type="application/x-java-object"/>
        <value media-type="application/x-java-object"/>
      </encoding>
      <memory max-count="10000"/>
    </local-cache>
    <distributed-cache name="sessions" owners="2">
      <expiration lifespan="-1"/>
    </distributed-cache>
    <distributed-cache name="authenticationSessions" owners="2">
      <expiration lifespan="-1"/>
    </distributed-cache>
    <distributed-cache name="offlineSessions" owners="2">
      <expiration lifespan="-1"/>
    </distributed-cache>
    <distributed-cache name="clientSessions" owners="2">
      <expiration lifespan="-1"/>
    </distributed-cache>
    <distributed-cache name="offlineClientSessions" owners="2">
      <expiration lifespan="-1"/>
    </distributed-cache>
    <distributed-cache name="loginFailures" owners="2">
      <expiration lifespan="-1"/>
    </distributed-cache>
    <local-cache name="authorization">
      <encoding>
        <key media-type="application/x-java-object"/>
        <value media-type="application/x-java-object"/>
      </encoding>
      <memory max-count="10000"/>
    </local-cache>
    <replicated-cache name="work">
      <expiration lifespan="-1"/>
    </replicated-cache>
    <local-cache name="keys">
      <encoding>
        <key media-type="application/x-java-object"/>
        <value media-type="application/x-java-object"/>
      </encoding>
      <expiration max-idle="3600000"/>
      <memory max-count="1000"/>
    </local-cache>
    <distributed-cache name="actionTokens" owners="2">
      <encoding>
        <key media-type="application/x-java-object"/>
        <value media-type="application/x-java-object"/>
      </encoding>
      <expiration max-idle="-1" lifespan="-1" interval="300000"/>
      <memory max-count="-1"/>
    </distributed-cache>
  </cache-container>
</infinispan>

docker-compose.yml

version: '3'

volumes:
  postgres_data:
    driver: local

services:
  postgres:
    image: postgres:11
    volumes:
      - postgres_data:/var/lib/postgresql/data
    environment:
      POSTGRES_DB: keycloak
      POSTGRES_USER: keycloak
      POSTGRES_PASSWORD: password
    ports:
      - 5433:5432
  keycloak:
    build:
      context: "./keycloakx"
      dockerfile: "./Dockerfile"
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: admin
      KC_DB_URL_HOST: postgres
      KC_DB_URL_DATABASE: keycloak
      KC_DB_SCHEMA: public
      KC_DB_USERNAME: keycloak
      KC_DB_PASSWORD: password
      KC_HOSTNAME_STRICT: false
      KC_HTTP_ENABLED: true
      KC_LOG_LEVEL: INFO,org.infinispan:DEBUG,org.jgroups:DEBUG
    ports:
      - 8080:8080
      - 8443:8443
    depends_on:
      - postgres

Dockerfile

FROM quay.io/keycloak/keycloak:17.0.0 as builder

ENV KC_METRICS_ENABLED=true
ENV KC_FEATURES=preview
ENV KC_DB=postgres
ENV KC_HTTP_RELATIVE_PATH=/auth
# specify the custom cache config file here
ENV KC_CACHE_CONFIG_FILE=cache-ispn-jdbc-ping.xml

# copy the custom cache config file into the keycloak conf dir
COPY ./cache-ispn-jdbc-ping.xml /opt/keycloak/conf/cache-ispn-jdbc-ping.xml

RUN /opt/keycloak/bin/kc.sh build

FROM quay.io/keycloak/keycloak:17.0.0
COPY --from=builder /opt/keycloak/lib/quarkus/ /opt/keycloak/lib/quarkus/
WORKDIR /opt/keycloak
# for demonstration purposes only, please make sure to use proper certificates in production instead
RUN keytool -genkeypair -storepass password -storetype PKCS12 -keyalg RSA -keysize 2048 -dname "CN=server" -alias server -ext "SAN:c=DNS:localhost,IP:127.0.0.1" -keystore conf/server.keystore

ENTRYPOINT ["/opt/keycloak/bin/kc.sh", "start"]

Hi,

Based on my teams configurations. I can see, insert_single_sql is missing in your jgroup stack details. Because of that i don't see any record in a jgroupsping table. I have added the below updated configuration from my end and can see the same at DB.

<JDBC_PING connection_driver="org.postgresql.Driver"
connection_username="${env.KC_DB_USERNAME}" connection_password="${env.KC_DB_PASSWORD}"
connection_url="${env.KC_DB_URL}"
initialize_sql="CREATE TABLE IF NOT EXISTS JGROUPSPING (own_addr varchar(200) NOT NULL, bind_addr VARCHAR(200) NOT NULL, created timestamp NOT NULL, cluster_name varchar(200) NOT NULL, ping_data BYTEA, constraint PK_JGROUPSPING PRIMARY KEY (own_addr, cluster_name));"
insert_single_sql="INSERT INTO JGROUPSPING (own_addr, bind_addr, created, cluster_name, ping_data) values (?,'${jboss.bind.address:127.0.0.1}',NOW(), ?, ?);"
delete_single_sql="DELETE FROM JGROUPSPING WHERE own_addr=? AND cluster_name=?;"
select_all_pingdata_sql="SELECT ping_data FROM JGROUPSPING WHERE cluster_name=?;"
info_writer_sleep_time="500"
remove_all_data_on_view_change="true"
stack.combine="REPLACE"
stack.position="MPING" />

Hi @Arulaln-AR @xgp ,
I need your help in configuring keycloak cluster. I'm using the custom stack for JDBC-PING to discover the instances, but its not working.
I have a shared Database and running two separate keycloak instances in docker in two different EC2 instances. Do i need to the IP address of atleast one instance for it to discover?
something like this or in the jdbc-custom stack. Any help on this would be much appreciated.

#IP address of this host, please make sure this IP can be accessed by the other Keycloak instances
JGROUPS_DISCOVERY_EXTERNAL_IP=172.21.48.39
#protocol
JGROUPS_DISCOVERY_PROTOCOL=JDBC_PING

Thank you!

Hi @Arulaln-AR
I refer to your configuration, after the service started successfully, there is a data in the table, but ping_data is empty. Could you help me?
Use keycloak version 22.0.1. Thanks.

Hi,
We have been using this configuration for our product for a while.
Lately we noticed this configuration will be printed out to the logs exposing the credentials.
This issue comes from infinispan, they just append the raw configuration to the exception's message (https://github.com/infinispan/infinispan/blob/3b7016b63b62b063eb649dd49a5de658ac78e890/core/src/main/java/org/infinispan/remoting/transport/jgroups/JGroupsTransport.java#L692)

ERROR [org.keycloak.quarkus.runtime.cli.ExecutionExceptionHandler] (main) ERROR: org.infinispan.manager.EmbeddedCacheManagerStartupException: ISPN000541: Error while trying to create a channel using the specified configuration '[TCP(bundler.max_size=64000, sock_conn_timeout=300, linger=-1, thread_pool.keep_alive_time=60000, diag.enabled=false, bind_port=7600, thread_naming_pattern=pl, non_blocking_sends=false, thread_pool.thread_dumps_threshold=10000, send_buf_size=640k, thread_pool.max_threads=200, use_virtual_threads=false, bundler_type=transfer-queue, external_addr=..., thread_pool.min_threads=0), RED(), JDBC_PING(select_all_pingdata_sql=SELECT ping_data, own_addr, cluster_name FROM JGROUPSPING WHERE cluster_name=?;, connection_password=<the password>, connection_url=<the url>, initialize_sql=CREATE TABLE IF NOT EXISTS JGROUPSPING (own_addr varchar(200) NOT NULL, cluster_name varchar(200) NOT NULL, bind_addr varchar(200) NOT NULL, updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, ping_data varbinary(5000) DEFAULT NULL, PRIMARY KEY (own_addr, cluster_name)) ENGINE=InnoDB DEFAULT CHARSET=utf8;, insert_single_sql=INSERT INTO JGROUPSPING (own_addr, cluster_name, bind_addr, updated, ping_data) values (?, ?, '...:7600', NOW(), ?);, connection_driver=com.mysql.jdbc.Driver, delete_single_sql=DELETE FROM JGROUPSPING WHERE own_addr=? AND cluster_name=?;, info_writer_sleep_time=500, connection_username=sso_proxy_keycloack, remove_all_data_on_view_change=true), MERGE3(max_interval=30000, min_interval=10000), FD_SOCK2(offset=50000), FD_ALL3(), VERIFY_SUSPECT2(timeout=1000), pbcast.NAKACK2(xmit_interval=200, xmit_table_num_rows=50, resend_last_seqno=true, use_mcast_xmit=false, xmit_table_msgs_per_row=1024, xmit_table_max_compaction_time=30000), UNICAST3(conn_close_timeout=5000, xmit_interval=200, xmit_table_num_rows=50, xmit_table_msgs_per_row=1024, xmit_table_max_compaction_time=30000), pbcast.STABLE(desired_avg_gossip=5000, max_bytes=1M), pbcast.GMS(join_timeout=2000, print_local_addr=false), UFC(min_threshold=0.40, max_credits=4m), MFC(min_threshold=0.40, max_credits=4m), FRAG4(frag_size=60000)]'

@vikviksecuri This is a great catch. However, it's not something that can be fixed via the Dockerfile or even in Keycloak. Did you open a ticket on the Infinispan project? If you could do that and link it here, we can make the fix in Infinispan logging and PR.