Complete TODO.md roadmap implementation: End-to-end FlinkDotNet streaming pipeline with IR Runner and Gateway

.github/workflows/build-ir-runner.yml

@@ -0,0 +1,142 @@
+name: Build IR Runner JAR
+
+on:
+  push:
+    branches: [ main, develop ]
+    paths:
+      - 'FlinkDotNet/Flink.IRRunner/**'
+      - '.github/workflows/build-ir-runner.yml'
+  pull_request:
+    branches: [ main, develop ]
+    paths:
+      - 'FlinkDotNet/Flink.IRRunner/**'
+      - '.github/workflows/build-ir-runner.yml'
+  workflow_dispatch:
+
+env:
+  GRADLE_OPTS: -Dorg.gradle.daemon=false -Xmx2g -Xms1g
+
+jobs:
+  build-ir-runner:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up JDK 17
+      uses: actions/setup-java@v4
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+
+    - name: Setup Gradle
+      uses: gradle/[email protected]
+
+    - name: Cache Gradle packages
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-
+
+    - name: Make gradlew executable
+      run: |
+        if [ ! -f FlinkDotNet/Flink.IRRunner/gradlew ]; then
+          cd FlinkDotNet/Flink.IRRunner && gradle wrapper --gradle-version=8.0
+        fi
+        chmod +x FlinkDotNet/Flink.IRRunner/gradlew
+
+    - name: Build IR Runner JAR
+      working-directory: FlinkDotNet/Flink.IRRunner
+      run: |
+        ./gradlew clean build
+        ./gradlew fatJar --no-daemon
+
+    - name: Test IR Runner
+      working-directory: FlinkDotNet/Flink.IRRunner
+      run: |
+        ./gradlew test --no-daemon
+
+    - name: Verify JAR exists
+      run: |
+        ls -la FlinkDotNet/Flink.IRRunner/build/libs/
+        test -f FlinkDotNet/Flink.IRRunner/build/libs/flink-ir-runner-1.0.0.jar
+
+    - name: Test JAR execution
+      working-directory: FlinkDotNet/Flink.IRRunner
+      run: |
+        # Create a simple test IR JSON
+        echo '{"metadata":{"jobId":"test","jobName":"Test Job","version":"1.0.0"},"source":{"type":"kafka","topic":"test-input"},"operations":[],"sink":{"type":"console"}}' > test-ir.json
+
+        # Test the JAR with --help flag
+        java -jar build/libs/flink-ir-runner-1.0.0.jar --help
+
+        # Test IR file loading (will fail gracefully without Flink cluster)
+        java -jar build/libs/flink-ir-runner-1.0.0.jar --ir-file test-ir.json || true
+
+    - name: Upload IR Runner JAR artifact
+      uses: actions/upload-artifact@v3
+      with:
+        name: flink-ir-runner-jar
+        path: FlinkDotNet/Flink.IRRunner/build/libs/flink-ir-runner-1.0.0.jar
+        retention-days: 30
+
+    - name: Calculate JAR checksums
+      run: |
+        cd FlinkDotNet/Flink.IRRunner/build/libs/
+        sha256sum flink-ir-runner-1.0.0.jar > flink-ir-runner-1.0.0.jar.sha256
+        md5sum flink-ir-runner-1.0.0.jar > flink-ir-runner-1.0.0.jar.md5
+
+    - name: Upload checksums
+      uses: actions/upload-artifact@v3
+      with:
+        name: flink-ir-runner-checksums
+        path: |
+          FlinkDotNet/Flink.IRRunner/build/libs/flink-ir-runner-1.0.0.jar.sha256
+          FlinkDotNet/Flink.IRRunner/build/libs/flink-ir-runner-1.0.0.jar.md5
+        retention-days: 30
+
+    - name: Report build metrics
+      run: |
+        JAR_SIZE=$(stat -c%s FlinkDotNet/Flink.IRRunner/build/libs/flink-ir-runner-1.0.0.jar)
+        echo "IR Runner JAR size: $JAR_SIZE bytes"
+        echo "IR Runner JAR size: $(($JAR_SIZE / 1024 / 1024)) MB"
+
+        # Check for essential classes in the JAR
+        jar tf FlinkDotNet/Flink.IRRunner/build/libs/flink-ir-runner-1.0.0.jar | grep -E "(FlinkIRRunner|jackson|flink)" | head -10 || true
+
+  release-artifact:
+    needs: build-ir-runner
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+
+    steps:
+    - name: Download IR Runner JAR
+      uses: actions/download-artifact@v3
+      with:
+        name: flink-ir-runner-jar
+        path: ./artifacts/
+
+    - name: Download checksums
+      uses: actions/download-artifact@v3
+      with:
+        name: flink-ir-runner-checksums
+        path: ./artifacts/
+
+    - name: Create GitHub Release (if tag)
+      if: startsWith(github.ref, 'refs/tags/')
+      uses: softprops/action-gh-release@v1
+      with:
+        files: |
+          ./artifacts/flink-ir-runner-1.0.0.jar
+          ./artifacts/flink-ir-runner-1.0.0.jar.sha256
+          ./artifacts/flink-ir-runner-1.0.0.jar.md5
+        draft: false
+        prerelease: false
+        generate_release_notes: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -103,3 +103,25 @@ apphost_test.log
 kafka_2.13-4.0.0/
 NativeKafkaBridge/libnativekafkabridge.so
 .roo/mcp.json
+
+# Gradle build files
+.gradle/
+build/
+!gradle/wrapper/gradle-wrapper.jar
+gradle/wrapper/gradle-wrapper.properties
+gradlew
+gradlew.bat
+
+# Java build artifacts
+*.class
+*.jar
+*.war
+*.ear
+*.tar
+*.zip
+
+# IDE files
+.idea/
+*.iml
+*.iws
+*.ipr

FlinkDotNet/Flink.IRRunner/build.gradle

@@ -0,0 +1,95 @@
+plugins {
+    id 'java'
+    id 'application'
+}
+
+group = 'com.flinkdotnet'
+version = '1.0.0'
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_11
+    targetCompatibility = JavaVersion.VERSION_11
+}
+
+repositories {
+    mavenCentral()
+}
+
+// Flink 1.18.1 dependencies (stable)
+def flinkVersion = '1.18.1'
+def jacksonVersion = '2.15.2'
+def kafkaVersion = '3.5.1'
+
+dependencies {
+    // Flink Core
+    implementation "org.apache.flink:flink-streaming-java:${flinkVersion}"
+    implementation "org.apache.flink:flink-java:${flinkVersion}"
+    implementation "org.apache.flink:flink-clients:${flinkVersion}"
+
+    // Flink Connectors
+    implementation "org.apache.flink:flink-connector-kafka:3.0.2-1.18"
+    implementation "org.apache.flink:flink-connector-files:${flinkVersion}"
+
+    // JSON Processing
+    implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}"
+    implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}"
+    implementation "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}"
+    implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${jacksonVersion}"
+
+    // Kafka Client (for connectors)
+    implementation "org.apache.kafka:kafka-clients:${kafkaVersion}"
+
+    // Logging
+    implementation 'org.slf4j:slf4j-api:2.0.7'
+    implementation 'org.apache.logging.log4j:log4j-slf4j2-impl:2.20.0'
+    implementation 'org.apache.logging.log4j:log4j-core:2.20.0'
+
+    // Testing
+    testImplementation 'junit:junit:4.13.2'
+    testImplementation "org.apache.flink:flink-test-utils:${flinkVersion}"
+    testImplementation "org.apache.flink:flink-streaming-java:${flinkVersion}:tests"
+}
+
+application {
+    mainClass = 'com.flinkdotnet.irrunner.FlinkIRRunner'
+}
+
+// Create fat JAR with all dependencies manually
+task fatJar(type: Jar) {
+    archiveBaseName = 'flink-ir-runner'
+    from { configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) } }
+    with jar
+
+    manifest {
+        attributes(
+            'Main-Class': 'com.flinkdotnet.irrunner.FlinkIRRunner',
+            'Implementation-Title': 'FlinkDotNet IR Runner',
+            'Implementation-Version': version
+        )
+    }
+
+    // Exclude conflicting META-INF files
+    exclude 'META-INF/*.SF'
+    exclude 'META-INF/*.DSA'
+    exclude 'META-INF/*.RSA'
+
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
+
+// Configure Java compilation
+compileJava {
+    options.encoding = 'UTF-8'
+    options.compilerArgs << '-parameters'
+}
+
+tasks.withType(JavaCompile) {
+    options.warnings = false
+}
+
+// Ensure fat jar is built by default but fix dependencies
+build.dependsOn fatJar
+
+// Fix distribution task dependencies
+distTar.dependsOn fatJar
+distZip.dependsOn fatJar
+startScripts.dependsOn fatJar

FlinkDotNet/Flink.IRRunner/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.0-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

FlinkDotNet/Flink.IRRunner/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'flink-ir-runner'