diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml
new file mode 100644
index 0000000..dd14bf3
--- /dev/null
+++ b/.github/workflows/matrix.yml
@@ -0,0 +1,29 @@
+name: Compatibility Matrix
+
+on:
+  workflow_dispatch:
+
+jobs:
+  compatibility:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        keycloak_version: [ 21.0.2, 21.1.2, 22.0.5, 23.0.5, latest, nightly ]
+        extension_version: [ 20.0.1, 21.0.0, 22.0.0, 23.0.0 ]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            ref: v${{ matrix.extension_version }}
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17
+          cache: 'maven'
+      - name: Clean and compile
+        run: mvn -B -U clean test-compile --file pom.xml
+      - name: Download extension
+        run: curl -L -o target/keycloak-restrict-client-auth.jar https://github.com/sventorben/keycloak-restrict-client-auth/releases/download/v${{ matrix.extension_version }}/keycloak-restrict-client-auth.jar
+      - name: Compatibility tests
+        run: mvn -B -U failsafe:integration-test failsafe:verify --file pom.xml -Dkeycloak.version=${{ matrix.keycloak_version }} -DuseJar=true
diff --git a/pom.xml b/pom.xml
index 3fc8c71..aa495a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,6 +56,7 @@
         <!-- For compatibility tests -->
         <keycloak.version>${version.keycloak}</keycloak.version>
         <keycloak.dist>quarkus</keycloak.dist>
+        <useJar>false</useJar>
 
         <version.mockito>5.10.0</version.mockito>
     </properties>
@@ -87,6 +88,7 @@
                             <org.jboss.logging.provider>log4j2</org.jboss.logging.provider>
                             <keycloak.version>${keycloak.version}</keycloak.version>
                             <keycloak.dist>${keycloak.dist}</keycloak.dist>
+                            <useJar>${useJar}</useJar>
                         </systemPropertyVariables>
                     </configuration>
                 </plugin>
diff --git a/src/test/java/de/sventorben/keycloak/authorization/client/ClientPolicyIT.java b/src/test/java/de/sventorben/keycloak/authorization/client/ClientPolicyIT.java
index 9589929..61c1457 100755
--- a/src/test/java/de/sventorben/keycloak/authorization/client/ClientPolicyIT.java
+++ b/src/test/java/de/sventorben/keycloak/authorization/client/ClientPolicyIT.java
@@ -34,7 +34,6 @@ class ClientPolicyIT {
 
     @Container
     private static final KeycloakContainer KEYCLOAK_CONTAINER = FullImageName.createContainer()
-        .withProviderClassesFrom("target/classes")
         .withExposedPorts(KEYCLOAK_HTTP_PORT)
         .withLogConsumer(new Slf4jLogConsumer(LOGGER).withSeparateOutputStreams())
         .withRealmImportFile("/test-realm.json")
diff --git a/src/test/java/de/sventorben/keycloak/authorization/client/ConfigIT.java b/src/test/java/de/sventorben/keycloak/authorization/client/ConfigIT.java
index 67631bc..8bcc44d 100755
--- a/src/test/java/de/sventorben/keycloak/authorization/client/ConfigIT.java
+++ b/src/test/java/de/sventorben/keycloak/authorization/client/ConfigIT.java
@@ -26,7 +26,6 @@ class ConfigIT {
 
     @Container
     private static final KeycloakContainer KEYCLOAK_CONTAINER = FullImageName.createContainer()
-        .withProviderClassesFrom("target/classes")
         .withExposedPorts(KEYCLOAK_HTTP_PORT)
         .withLogConsumer(new Slf4jLogConsumer(LOGGER).withSeparateOutputStreams())
         .withStartupTimeout(Duration.ofSeconds(90))
diff --git a/src/test/java/de/sventorben/keycloak/authorization/client/FullImageName.java b/src/test/java/de/sventorben/keycloak/authorization/client/FullImageName.java
index 5bd75d7..48122d6 100755
--- a/src/test/java/de/sventorben/keycloak/authorization/client/FullImageName.java
+++ b/src/test/java/de/sventorben/keycloak/authorization/client/FullImageName.java
@@ -4,6 +4,9 @@ import dasniko.testcontainers.keycloak.KeycloakContainer;
 import org.testcontainers.images.ImagePullPolicy;
 import org.testcontainers.images.PullPolicy;
 
+import java.io.File;
+import java.util.List;
+
 import static de.sventorben.keycloak.authorization.client.FullImageName.Distribution.quarkus;
 import static java.lang.module.ModuleDescriptor.Version;
 
@@ -20,6 +23,8 @@ class FullImageName {
     private static final String NIGHTLY_VERSION = "nightly";
     private static final String KEYCLOAK_VERSION = System.getProperty("keycloak.version", LATEST_VERSION);
 
+    private static final boolean USE_JAR = Boolean.parseBoolean(System.getProperty("useJar", "false"));
+
     static String get() {
         String imageName = "keycloak";
 
@@ -61,8 +66,14 @@ class FullImageName {
         if (isLatestVersion() || isNightlyVersion()) {
             pullPolicy = PullPolicy.alwaysPull();
         }
-        return new KeycloakContainer(fullImage)
+        KeycloakContainer keycloakContainer = new KeycloakContainer(fullImage)
             .withImagePullPolicy(pullPolicy);
+        if (USE_JAR) {
+            keycloakContainer = keycloakContainer.withProviderLibsFrom(List.of(new File("target/keycloak-restrict-client-auth.jar")));
+        } else {
+            keycloakContainer = keycloakContainer.withProviderClassesFrom("target/classes");
+        }
+        return keycloakContainer;
     }
 
 }
diff --git a/src/test/java/de/sventorben/keycloak/authorization/client/LoginIT.java b/src/test/java/de/sventorben/keycloak/authorization/client/LoginIT.java
index 1f97c57..2758ea3 100755
--- a/src/test/java/de/sventorben/keycloak/authorization/client/LoginIT.java
+++ b/src/test/java/de/sventorben/keycloak/authorization/client/LoginIT.java
@@ -33,7 +33,6 @@ class LoginIT {
 
     @Container
     private static final KeycloakContainer KEYCLOAK_CONTAINER = FullImageName.createContainer()
-        .withProviderClassesFrom("target/classes")
         .withExposedPorts(KEYCLOAK_HTTP_PORT)
         .withLogConsumer(new Slf4jLogConsumer(LOGGER).withSeparateOutputStreams())
         .withRealmImportFile("/test-realm.json")