3 Commits

Author SHA1 Message Date
91c10c89da feat(crd): add dual-format Args field supporting map and list
Add a custom Args type that accepts both a key-value map and a flat
CLI-style list, normalizing internally to []string so downstream code
requires no changes.

- Args: new []string alias with UnmarshalJSON (map + list) and
  MarshalJSON (map) — keys are sorted for deterministic ordering
- CRD schema: switch args from typed array to
  x-kubernetes-preserve-unknown-fields to accept either shape
- Bump go-flink-client to v0.2.2 (fix: programArgsList → programArg
  query parameter name for Flink REST API)

Args can now be written as:
  args:
    kafka-host: broker:9092
    kafka-group-id: my-group

Or the existing list format:
  args:
    - "--kafka-host"
    - broker:9092
2026-07-23 15:25:08 +03:30
7fb18cd45f feat: support multiple operator in single namespace 2026-07-23 13:51:33 +03:30
494d32c565 feat: update flink to 2.3 2026-07-22 10:48:53 +03:30
19 changed files with 252 additions and 47 deletions

View File

@@ -1,32 +1,73 @@
FROM public.ecr.aws/docker/library/flink:1.20.1-scala_2.12-java17 ARG FLINK_VERSION=2.3.0
ARG SCALA_VERSION=2.12
ARG JAVA_VERSION=21
FROM flink:${FLINK_VERSION}-scala_${SCALA_VERSION}-java${JAVA_VERSION}
# Set working directory # Set working directory
WORKDIR /opt/flink WORKDIR /opt/flink
# Set environment variables for Flink mini-cluster # Set environment variables for Flink mini-cluster
ENV FLINK_HOME /opt/flink ENV FLINK_HOME=/opt/flink
ENV PATH=$FLINK_HOME/bin:$PATH ENV PATH=$FLINK_HOME/bin:$PATH
# Expose necessary ports for the Flink UI (JobManager) and job manager # Expose necessary ports for the Flink UI (JobManager) and job manager
EXPOSE 8081 6123 EXPOSE 8081 6123
COPY ./start-cluster.sh /opt/flink/bin/start-cluster.sh COPY ./start-cluster.sh /opt/flink/bin/start-cluster.sh
RUN chmod +x /opt/flink/bin/start-cluster.sh
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-connector-kafka/3.4.0-1.20/flink-connector-kafka-3.4.0-1.20.jar -P /opt/flink/lib/ # ---- Flink-version-dependent connectors ----
# Kafka connector
ARG KAFKA_CONNECTOR_VERSION=5.0.0-2.2
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-connector-kafka/${KAFKA_CONNECTOR_VERSION}/flink-connector-kafka-${KAFKA_CONNECTOR_VERSION}.jar -P /opt/flink/lib/
# Kafka clients (version-independent)
RUN wget -q https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.9.0/kafka-clients-3.9.0.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.9.0/kafka-clients-3.9.0.jar -P /opt/flink/lib/
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-avro/1.20.1/flink-avro-1.20.1.jar -P /opt/flink/lib/
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-avro-confluent-registry/1.20.1/flink-avro-confluent-registry-1.20.1.jar -P /opt/flink/lib/ # Avro format
ARG AVRO_VERSION=2.2.1
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-avro/${AVRO_VERSION}/flink-avro-${AVRO_VERSION}.jar -P /opt/flink/lib/
# Avro Confluent Registry
ARG AVRO_REGISTRY_VERSION=2.2.1
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-avro-confluent-registry/${AVRO_REGISTRY_VERSION}/flink-avro-confluent-registry-${AVRO_REGISTRY_VERSION}.jar -P /opt/flink/lib/
# ---- Version-independent connectors ----
# ClickHouse SQL connector
RUN wget -q https://repo1.maven.org/maven2/name/nkonev/flink/flink-sql-connector-clickhouse/1.17.1-8/flink-sql-connector-clickhouse-1.17.1-8.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/name/nkonev/flink/flink-sql-connector-clickhouse/1.17.1-8/flink-sql-connector-clickhouse-1.17.1-8.jar -P /opt/flink/lib/
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-postgres-cdc/3.2.1/flink-sql-connector-postgres-cdc-3.2.1.jar -P /opt/flink/lib/
# PostgreSQL CDC
ARG PG_CDC_VERSION=3.6.0-2.2
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-postgres-cdc/${PG_CDC_VERSION}/flink-sql-connector-postgres-cdc-${PG_CDC_VERSION}.jar -P /opt/flink/lib/
# Legacy Avro
RUN wget -q https://repo1.maven.org/maven2/org/apache/avro/avro/1.8.2/avro-1.8.2.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/org/apache/avro/avro/1.8.2/avro-1.8.2.jar -P /opt/flink/lib/
# misc libs
RUN wget -q https://repo1.maven.org/maven2/net/objecthunter/exp4j/0.4.5/exp4j-0.4.5.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/net/objecthunter/exp4j/0.4.5/exp4j-0.4.5.jar -P /opt/flink/lib/
RUN wget -q https://jdbc.postgresql.org/download/postgresql-42.7.4.jar -P /opt/flink/lib/ RUN wget -q https://jdbc.postgresql.org/download/postgresql-42.7.4.jar -P /opt/flink/lib/
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-sql-jdbc-driver/1.20.1/flink-sql-jdbc-driver-1.20.1.jar -P /opt/flink/lib/
# JDBC driver
ARG JDBC_DRIVER_VERSION=2.2.1
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-sql-jdbc-driver/${JDBC_DRIVER_VERSION}/flink-sql-jdbc-driver-${JDBC_DRIVER_VERSION}.jar -P /opt/flink/lib/
# Legacy Scala JDBC connector (no 2.x version exists)
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-jdbc_2.12/1.10.3/flink-jdbc_2.12-1.10.3.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-jdbc_2.12/1.10.3/flink-jdbc_2.12-1.10.3.jar -P /opt/flink/lib/
# JDBC connector (latest: 3.3.0-1.20, no 2.x version yet)
ARG JDBC_CONNECTOR_VERSION=3.3.0-1.20
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/${JDBC_CONNECTOR_VERSION}/flink-connector-jdbc-${JDBC_CONNECTOR_VERSION}.jar -P /opt/flink/lib/
# misc libs
RUN wget -q https://repo1.maven.org/maven2/com/aventrix/jnanoid/jnanoid/2.0.0/jnanoid-2.0.0.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/com/aventrix/jnanoid/jnanoid/2.0.0/jnanoid-2.0.0.jar -P /opt/flink/lib/
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/3.2.0-1.19/flink-connector-jdbc-3.2.0-1.19.jar -P /opt/flink/lib/
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-presto/1.20.1/flink-s3-fs-presto-1.20.1.jar -P /opt/flink/lib/ # S3 filesystem
ARG S3_FS_VERSION=2.2.1
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-presto/${S3_FS_VERSION}/flink-s3-fs-presto-${S3_FS_VERSION}.jar -P /opt/flink/lib/
# Compression
RUN wget -q https://repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.7-2/zstd-jni-1.5.7-2.jar -P /opt/flink/lib/ RUN wget -q https://repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.7-2/zstd-jni-1.5.7-2.jar -P /opt/flink/lib/
# Command to start Flink JobManager and TaskManager in a mini-cluster setup # Command to start Flink JobManager and TaskManager in a mini-cluster setup

View File

@@ -2,9 +2,9 @@
apiVersion: apiextensions.k8s.io/v1 apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition kind: CustomResourceDefinition
metadata: metadata:
name: flink-jobs.flink.logicamp.tech name: flink-jobs.flink.logicamp.dev
spec: spec:
group: flink.logicamp.tech group: flink.logicamp.dev
names: names:
kind: FlinkJob kind: FlinkJob
plural: flink-jobs plural: flink-jobs
@@ -30,6 +30,9 @@ spec:
type: string type: string
name: name:
type: string type: string
flinkCluster:
type: string
description: "Identifier of the Flink cluster that manages this job. Defaults to the OPERATOR_ID (usually the Helm release name) if not set."
entryClass: entryClass:
type: string type: string
parallelism: parallelism:
@@ -41,9 +44,8 @@ spec:
jarURIBasicAuthPassword: jarURIBasicAuthPassword:
type: string type: string
args: args:
type: array description: "Program arguments. Accepts a list of strings (--key, value, ...) or a map of key-value pairs."
items: x-kubernetes-preserve-unknown-fields: true
type: string
savepointInterval: savepointInterval:
type: string type: string
format: duration format: duration

View File

@@ -1,5 +1,5 @@
# flink-job-instance.yaml # flink-job-instance.yaml
apiVersion: flink.logicamp.tech/v1alpha1 apiVersion: flink.logicamp.dev/v1alpha1
kind: FlinkJob kind: FlinkJob
metadata: metadata:
name: my-flink-job name: my-flink-job
@@ -11,6 +11,17 @@ spec:
jarUri: "https://git.logicamp.tech/api/packages/logiline/generic/facility-enrichment/1.0.0/facility-enrichment.jar" jarUri: "https://git.logicamp.tech/api/packages/logiline/generic/facility-enrichment/1.0.0/facility-enrichment.jar"
jarURIBasicAuthUsername: logiline-actrunner jarURIBasicAuthUsername: logiline-actrunner
jarURIBasicAuthPassword: daeweeb7ohpaiw3oojiCoong jarURIBasicAuthPassword: daeweeb7ohpaiw3oojiCoong
# args accepts two formats:
# Map (recommended):
args:
kafka-host: kafka.pg-release:9092
kafka-group-id: batch-sales-config-dedup-processor
# List (also valid):
# args:
# - "--kafka-host"
# - kafka.pg-release:9092
# - "--kafka-group-id"
# - batch-sales-config-dedup-processor
flinkConfiguration: flinkConfiguration:
taskmanager.numberOfTaskSlots: "1" taskmanager.numberOfTaskSlots: "1"
parallelism.default: "1" parallelism.default: "1"

2
go.mod
View File

@@ -5,7 +5,7 @@ go 1.23.2
require ( require (
github.com/danielgtaylor/huma/v2 v2.27.0 github.com/danielgtaylor/huma/v2 v2.27.0
github.com/gofiber/fiber/v2 v2.52.6 github.com/gofiber/fiber/v2 v2.52.6
github.com/logi-camp/go-flink-client v0.2.0 github.com/logi-camp/go-flink-client v0.2.2
github.com/samber/lo v1.47.0 github.com/samber/lo v1.47.0
go.uber.org/zap v1.27.0 go.uber.org/zap v1.27.0
k8s.io/apimachinery v0.31.3 k8s.io/apimachinery v0.31.3

6
go.sum
View File

@@ -62,10 +62,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/logi-camp/go-flink-client v0.2.0 h1:PIyfJq7FjW28bnvemReCicIuQD7JzVgJDk2xPTZUS2s= github.com/logi-camp/go-flink-client v0.2.2 h1:aWqibzcJiv02Myf/WowZafI0hhHp5x7BJGRCmI6PT20=
github.com/logi-camp/go-flink-client v0.2.0/go.mod h1:A79abedX6wGQI0FoICdZI7SRoGHj15QwMwWowgsKYFI= github.com/logi-camp/go-flink-client v0.2.2/go.mod h1:A79abedX6wGQI0FoICdZI7SRoGHj15QwMwWowgsKYFI=
github.com/logi-camp/go-flink-client v0.2.1 h1:STfKamFm9+2SxxfZO3ysdFsb5MViQdThB4UHbnkUOE8=
github.com/logi-camp/go-flink-client v0.2.1/go.mod h1:A79abedX6wGQI0FoICdZI7SRoGHj15QwMwWowgsKYFI=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=

View File

@@ -2,5 +2,5 @@ apiVersion: v2
name: flink-kube-operator name: flink-kube-operator
description: Helm chart for flink kube operator description: Helm chart for flink kube operator
type: application type: application
version: 1.2.3 version: 1.3.0
appVersion: "0.1.1" appVersion: "0.1.1"

View File

@@ -11,7 +11,11 @@
taskmanager.data.port: 6125 taskmanager.data.port: 6125
taskmanager.numberOfTaskSlots: {{ .Values.flink.taskManager.numberOfTaskSlots }} taskmanager.numberOfTaskSlots: {{ .Values.flink.taskManager.numberOfTaskSlots }}
parallelism.default: {{ .Values.flink.parallelism.default }} parallelism.default: {{ .Values.flink.parallelism.default }}
{{- if semverCompare ">=2.0.0" .Values.flink.version }}
state.backend.type: {{ .Values.flink.state.backend }}
{{- else }}
state.backend: {{ .Values.flink.state.backend }} state.backend: {{ .Values.flink.state.backend }}
{{- end }}
rest.port: 8081 rest.port: 8081
rootLogger.level = DEBUG rootLogger.level = DEBUG
rootLogger.appenderRef.console.ref = ConsoleAppender rootLogger.appenderRef.console.ref = ConsoleAppender
@@ -23,14 +27,18 @@
{{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }} {{- if eq .Values.flink.state.checkpoint.storageType "filesystem" }}
state.checkpoints.dir: file:///opt/flink/checkpoints/ state.checkpoints.dir: file:///opt/flink/checkpoints/
{{- else if eq .Values.flink.state.checkpoint.storageType "s3" }} {{- else if eq .Values.flink.state.checkpoint.storageType "s3" }}
state.checkpoints.dir: s3://flink/checkpoints/ state.checkpoints.dir: s3://{{ .Release.Name }}-flink/checkpoints/
{{- end }} {{- end }}
{{- if semverCompare ">=2.0.0" .Values.flink.version }}
state.backend.rocksdb.local-dir: /opt/flink/rocksdb
{{- else }}
state.backend.rocksdb.localdir: /opt/flink/rocksdb state.backend.rocksdb.localdir: /opt/flink/rocksdb
{{- end }}
high-availability.storageDir: /opt/flink/ha high-availability.storageDir: /opt/flink/ha
{{- if eq .Values.flink.state.savepoint.storageType "filesystem" }} {{- if eq .Values.flink.state.savepoint.storageType "filesystem" }}
state.savepoints.dir: file:///opt/flink/savepoints/ state.savepoints.dir: file:///opt/flink/savepoints/
{{- else if eq .Values.flink.state.savepoint.storageType "s3" }} {{- else if eq .Values.flink.state.savepoint.storageType "s3" }}
state.savepoints.dir: s3://flink/savepoints/ state.savepoints.dir: s3://{{ .Release.Name }}-flink/savepoints/
{{- end }} {{- end }}
state.backend.incremental: {{ .Values.flink.state.incremental }} state.backend.incremental: {{ .Values.flink.state.incremental }}
rest.profiling.enabled: true rest.profiling.enabled: true

View File

@@ -24,6 +24,7 @@ spec:
initContainers: initContainers:
- name: volume-mount-hack - name: volume-mount-hack
image: {{ .Values.flink.image.repository }}:{{ .Values.flink.image.tag }} image: {{ .Values.flink.image.repository }}:{{ .Values.flink.image.tag }}
securityContext:
runAsUser: 0 runAsUser: 0
command: ["sh", "-c", "chown -R flink {{ .Values.flink.state.ha.dir }}"] command: ["sh", "-c", "chown -R flink {{ .Values.flink.state.ha.dir }}"]
volumeMounts: volumeMounts:
@@ -51,17 +52,17 @@ spec:
fieldPath: status.podIP fieldPath: status.podIP
{{- if or (eq .Values.flink.state.checkpoint.storageType "s3") (eq .Values.flink.state.savepoint.storageType "s3") }} {{- if or (eq .Values.flink.state.checkpoint.storageType "s3") (eq .Values.flink.state.savepoint.storageType "s3") }}
- name: S3_ENDPOINT - name: S3_ENDPOINT
value: "http://minio-service:9000" value: "http://{{ .Release.Name }}-minio:9000"
- name: AWS_ACCESS_KEY_ID - name: AWS_ACCESS_KEY_ID
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
name: {{ .Release.Name }}-flink-secrets name: {{ .Release.Name }}-minio
key: minio_access_key key: root-user
- name: AWS_SECRET_ACCESS_KEY - name: AWS_SECRET_ACCESS_KEY
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
name: {{ .Release.Name }}-flink-secrets name: {{ .Release.Name }}-minio
key: minio_secret_key key: root-password
{{- end }} {{- end }}
volumeMounts: volumeMounts:
- name: flink-ha - name: flink-ha

View File

@@ -33,17 +33,17 @@ spec:
fieldPath: status.podIP fieldPath: status.podIP
{{- if or (eq .Values.flink.state.checkpoint.storageType "s3") (eq .Values.flink.state.savepoint.storageType "s3") }} {{- if or (eq .Values.flink.state.checkpoint.storageType "s3") (eq .Values.flink.state.savepoint.storageType "s3") }}
- name: S3_ENDPOINT - name: S3_ENDPOINT
value: "http://minio-service:9000" value: "http://{{ .Release.Name }}-minio:9000"
- name: AWS_ACCESS_KEY_ID - name: AWS_ACCESS_KEY_ID
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
name: {{ .Release.Name }}-flink-secrets name: {{ .Release.Name }}-minio
key: minio_access_key key: root-user
- name: AWS_SECRET_ACCESS_KEY - name: AWS_SECRET_ACCESS_KEY
valueFrom: valueFrom:
secretKeyRef: secretKeyRef:
name: {{ .Release.Name }}-flink-secrets name: {{ .Release.Name }}-minio
key: minio_secret_key key: root-password
{{- end }} {{- end }}
volumeMounts: volumeMounts:
- name: rocksdb-storage - name: rocksdb-storage

View File

@@ -8,8 +8,8 @@ metadata:
spec: spec:
scaleTargetRef: scaleTargetRef:
apiVersion: apps/v1 apiVersion: apps/v1
kind: Deployment kind: StatefulSet
name: {{ include "flink-kube-operator.fullname" . }} name: {{ .Release.Name }}-flink-operator
minReplicas: {{ .Values.autoscaling.minReplicas }} minReplicas: {{ .Values.autoscaling.minReplicas }}
maxReplicas: {{ .Values.autoscaling.maxReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }}
metrics: metrics:

View File

@@ -6,7 +6,7 @@ metadata:
{{- include "flink-kube-operator.labels" . | nindent 4 }} {{- include "flink-kube-operator.labels" . | nindent 4 }}
rules: rules:
- apiGroups: - apiGroups:
- flink.logicamp.tech # API group of the FlinkJob CRD - flink.logicamp.dev # API group of the FlinkJob CRD
resources: resources:
- flink-jobs # The plural name of your custom resource - flink-jobs # The plural name of your custom resource
verbs: verbs:

View File

@@ -44,13 +44,15 @@ spec:
containerPort: {{ .Values.service.port }} containerPort: {{ .Values.service.port }}
protocol: TCP protocol: TCP
env: env:
- name: OPERATOR_ID
value: {{ .Values.operatorId | default .Release.Name }}
- name: FLINK_API_URL - name: FLINK_API_URL
value: {{ .Release.Name }}-flink-job-manager:8081 value: {{ .Release.Name }}-flink-job-manager:8081
- name: NAMESPACE - name: NAMESPACE
value: "{{ .Release.Namespace }}" value: "{{ .Release.Namespace }}"
{{- if eq .Values.flink.state.savepoint.storageType "s3" }} {{- if eq .Values.flink.state.savepoint.storageType "s3" }}
- name: SAVEPOINT_PATH - name: SAVEPOINT_PATH
value: s3://flink/savepoints/ value: s3://{{ .Release.Name }}-flink/savepoints/
- name: S3_ENDPOINT - name: S3_ENDPOINT
value: "http://{{ .Release.Name }}-minio:9000" value: "http://{{ .Release.Name }}-minio:9000"
- name: AWS_ACCESS_KEY_ID - name: AWS_ACCESS_KEY_ID

View File

@@ -7,11 +7,11 @@ replicaCount: 1
# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
image: image:
repository: lcr.logicamp.tech/library/flink-kube-operator repository: logicampdev/flink-kube-operator
# This sets the pull policy for images. # This sets the pull policy for images.
pullPolicy: Always pullPolicy: Always
# Overrides the image tag whose default is the chart appVersion. # Overrides the image tag whose default is the chart appVersion.
tag: latest tag: v1.3.1
# This is for the secretes for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ # This is for the secretes for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
imagePullSecrets: [] imagePullSecrets: []
@@ -105,6 +105,11 @@ autoscaling:
config: config:
flinkApiUrl: flink:8081 flinkApiUrl: flink:8081
# Identifier for this operator instance. Used to partition FlinkJob CRs via
# label selector. Defaults to the Helm release name if left empty.
# Set explicitly when you need stable names across upgrades.
operatorId: ""
nodeSelector: {} nodeSelector: {}
tolerations: [] tolerations: []
@@ -113,9 +118,13 @@ affinity: {}
# Global values for the Flink deployment # Global values for the Flink deployment
flink: flink:
# Flink major.minor version (semver string). Used by config template to emit
# the correct configuration key names for the target Flink version.
# Override to "1.20" for backward compatibility with Flink 1.20.x.
version: "2.3"
image: image:
repository: lcr.logicamp.tech/library/flink repository: logicampdev/flink-kube-operator-flink
tag: 1.20.1-scala_2.12-java17-minicluster tag: 2.3.0-java21-v1.3.0
parallelism: parallelism:
default: 1 # Default parallelism for Flink jobs default: 1 # Default parallelism for Flink jobs

View File

@@ -1,7 +1,10 @@
package v1alpha1 package v1alpha1
import ( import (
"encoding/json"
"errors" "errors"
"sort"
"strings"
"time" "time"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -9,16 +12,64 @@ import (
//go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen object paths=$GOFILE //go:generate go run sigs.k8s.io/controller-tools/cmd/controller-gen object paths=$GOFILE
// Args is a list of program arguments passed to the Flink job.
// It accepts two formats in JSON/YAML:
//
// List: ["--kafka-host", "broker:9092", "--kafka-group-id", "my-group"]
// Map: {kafka-host: "broker:9092", kafka-group-id: "my-group"}
//
// Map keys are automatically prefixed with "--" and expanded into alternating
// key/value pairs. The internal representation is always a flat []string.
type Args []string
// UnmarshalJSON implements dual-format deserialization.
func (a *Args) UnmarshalJSON(data []byte) error {
// Try map format first: {"key": "value", ...}
var m map[string]string
if err := json.Unmarshal(data, &m); err == nil {
*a = make(Args, 0, len(m)*2)
// Sort keys for deterministic ordering.
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
*a = append(*a, "--"+k, m[k])
}
return nil
}
// Fall back to list format: ["--key", "value", ...]
var s []string
if err := json.Unmarshal(data, &s); err != nil {
return err
}
*a = Args(s)
return nil
}
// MarshalJSON serializes Args in the compact map format.
func (a Args) MarshalJSON() ([]byte, error) {
m := make(map[string]string, len(a)/2)
for i := 0; i < len(a)-1; i += 2 {
key := strings.TrimPrefix(a[i], "--")
m[key] = a[i+1]
}
return json.Marshal(m)
}
type FlinkJobSpec struct { type FlinkJobSpec struct {
Key string `json:"key"` Key string `json:"key"`
Name string `json:"name"` Name string `json:"name"`
FlinkCluster string `json:"flinkCluster"`
Parallelism int `json:"parallelism"` Parallelism int `json:"parallelism"`
JarURI string `json:"jarUri"` JarURI string `json:"jarUri"`
JarURIBasicAuthUsername *string `json:"jarURIBasicAuthUsername"` JarURIBasicAuthUsername *string `json:"jarURIBasicAuthUsername"`
JarURIBasicAuthPassword *string `json:"jarURIBasicAuthPassword"` JarURIBasicAuthPassword *string `json:"jarURIBasicAuthPassword"`
SavepointInterval metaV1.Duration `json:"savepointInterval"` SavepointInterval metaV1.Duration `json:"savepointInterval"`
EntryClass string `json:"entryClass"` EntryClass string `json:"entryClass"`
Args []string `json:"args"` Args Args `json:"args,omitempty"`
} }
type FlinkJobStatus struct { type FlinkJobStatus struct {

View File

@@ -6,7 +6,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/schema"
) )
const GroupName = "flink.logicamp.tech" const GroupName = "flink.logicamp.dev"
const GroupVersion = "v1alpha1" const GroupVersion = "v1alpha1"
const ResourceName = "flink-jobs" const ResourceName = "flink-jobs"

View File

@@ -3,6 +3,7 @@ package managed_job
import ( import (
"flink-kube-operator/internal/crd" "flink-kube-operator/internal/crd"
"flink-kube-operator/internal/crd/v1alpha1" "flink-kube-operator/internal/crd/v1alpha1"
"os"
"time" "time"
"flink-kube-operator/pkg" "flink-kube-operator/pkg"
@@ -73,6 +74,8 @@ func (mgr *Manager) cycle(client *api.Client, crdInstance *crd.Crd) {
} }
pkg.Logger.Debug("[manager] [cycle] overviews", zap.Any("overviews", jobManagerJobOverviews)) pkg.Logger.Debug("[manager] [cycle] overviews", zap.Any("overviews", jobManagerJobOverviews))
operatorId := os.Getenv("OPERATOR_ID")
// Loop over job definitions as Kubernetes CRD // Loop over job definitions as Kubernetes CRD
for _, uid := range crd.GetAllJobKeys() { for _, uid := range crd.GetAllJobKeys() {
if lo.Contains(mgr.processingJobsIds, uid) { if lo.Contains(mgr.processingJobsIds, uid) {
@@ -82,6 +85,27 @@ func (mgr *Manager) cycle(client *api.Client, crdInstance *crd.Crd) {
// Get job definition from Kubernetes CRD // Get job definition from Kubernetes CRD
def := crd.GetJob(uid) def := crd.GetJob(uid)
// Auto-claim: if flinkCluster is empty, set it to this operator's ID
if def.Spec.FlinkCluster == "" {
pkg.Logger.Info("[manager] auto-claiming unowned job", zap.String("name", def.GetName()))
crdInstance.Patch(def.GetUID(), map[string]interface{}{
"spec": map[string]interface{}{
"flinkCluster": operatorId,
},
})
def.Spec.FlinkCluster = operatorId
}
// Skip CRs that belong to another operator
if def.Spec.FlinkCluster != operatorId {
pkg.Logger.Debug("[manager] skipping job owned by another operator",
zap.String("name", def.GetName()),
zap.String("flinkCluster", def.Spec.FlinkCluster),
zap.String("operatorId", operatorId),
)
continue
}
mgr.processingJobsIds = append(mgr.processingJobsIds, uid) mgr.processingJobsIds = append(mgr.processingJobsIds, uid)
// Check if job exists in manager managed jobs // Check if job exists in manager managed jobs

View File

@@ -43,7 +43,7 @@ func (job *ManagedJob) Run(restoreMode bool) error {
EntryClass: job.def.Spec.EntryClass, EntryClass: job.def.Spec.EntryClass,
SavepointPath: savepointPath, SavepointPath: savepointPath,
Parallelism: job.def.Spec.Parallelism, Parallelism: job.def.Spec.Parallelism,
ProgramArg: job.def.Spec.Args, ProgramArgsList: job.def.Spec.Args,
}) })
if err == nil { if err == nil {
pkg.Logger.Info("[managed-job] [run] jar successfully ran", zap.Any("run-jar-resp", runJarResp)) pkg.Logger.Info("[managed-job] [run] jar successfully ran", zap.Any("run-jar-resp", runJarResp))

58
scripts/build-flink-images.sh Executable file
View File

@@ -0,0 +1,58 @@
#!/bin/bash
set -euo pipefail
REGISTRY="${REGISTRY:-lcr.logicamp.tech/library/flink}"
echo "=== Building Flink 2.3.0 image (default) ==="
docker build \
--build-arg FLINK_VERSION=2.3.0 \
--build-arg JAVA_VERSION=21 \
--build-arg KAFKA_CONNECTOR_VERSION=5.0.0-2.2 \
--build-arg AVRO_VERSION=2.2.1 \
--build-arg AVRO_REGISTRY_VERSION=2.2.1 \
--build-arg PG_CDC_VERSION=3.6.0-2.2 \
--build-arg JDBC_DRIVER_VERSION=2.2.1 \
--build-arg JDBC_CONNECTOR_VERSION=3.3.0-1.20 \
--build-arg S3_FS_VERSION=2.2.1 \
-f Dockerfile.flink \
-t "${REGISTRY}:2.3.0-scala_2.12-java21-minicluster" \
.
echo ""
echo "=== Building Flink 2.2.1 image ==="
docker build \
--build-arg FLINK_VERSION=2.2.1 \
--build-arg JAVA_VERSION=17 \
--build-arg KAFKA_CONNECTOR_VERSION=5.0.0-2.2 \
--build-arg AVRO_VERSION=2.2.1 \
--build-arg AVRO_REGISTRY_VERSION=2.2.1 \
--build-arg PG_CDC_VERSION=3.6.0-2.2 \
--build-arg JDBC_DRIVER_VERSION=2.2.1 \
--build-arg JDBC_CONNECTOR_VERSION=3.3.0-1.20 \
--build-arg S3_FS_VERSION=2.2.1 \
-f Dockerfile.flink \
-t "${REGISTRY}:2.2.1-scala_2.12-java17-minicluster" \
.
echo ""
echo "=== Building Flink 1.20.1 image (backward compat) ==="
docker build \
--build-arg FLINK_VERSION=1.20.1 \
--build-arg JAVA_VERSION=17 \
--build-arg KAFKA_CONNECTOR_VERSION=3.4.0-1.20 \
--build-arg AVRO_VERSION=1.20.1 \
--build-arg AVRO_REGISTRY_VERSION=1.20.1 \
--build-arg PG_CDC_VERSION=3.2.1 \
--build-arg JDBC_DRIVER_VERSION=1.20.1 \
--build-arg JDBC_CONNECTOR_VERSION=3.2.0-1.19 \
--build-arg S3_FS_VERSION=1.20.1 \
-f Dockerfile.flink \
-t "${REGISTRY}:1.20.1-scala_2.12-java17-minicluster" \
.
echo ""
echo "=== Done ==="
echo "Images built:"
echo " ${REGISTRY}:2.3.0-scala_2.12-java21-minicluster"
echo " ${REGISTRY}:2.2.1-scala_2.12-java17-minicluster"
echo " ${REGISTRY}:1.20.1-scala_2.12-java17-minicluster"

0
start-cluster.sh Normal file → Executable file
View File