74 lines
3.3 KiB
Docker
74 lines
3.3 KiB
Docker
ARG FLINK_VERSION=2.3.0
|
|
ARG SCALA_VERSION=2.12
|
|
ARG JAVA_VERSION=21
|
|
|
|
FROM flink:${FLINK_VERSION}-scala_${SCALA_VERSION}-java${JAVA_VERSION}
|
|
|
|
# Set working directory
|
|
WORKDIR /opt/flink
|
|
|
|
# Set environment variables for Flink mini-cluster
|
|
ENV FLINK_HOME=/opt/flink
|
|
ENV PATH=$FLINK_HOME/bin:$PATH
|
|
|
|
# Expose necessary ports for the Flink UI (JobManager) and job manager
|
|
EXPOSE 8081 6123
|
|
|
|
COPY ./start-cluster.sh /opt/flink/bin/start-cluster.sh
|
|
|
|
# ---- Flink-version-dependent connectors ----
|
|
|
|
# Kafka connector
|
|
ARG KAFKA_CONNECTOR_VERSION=5.0.0-2.2
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-connector-kafka/${KAFKA_CONNECTOR_VERSION}/flink-connector-kafka-${KAFKA_CONNECTOR_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# Kafka clients (version-independent)
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.9.0/kafka-clients-3.9.0.jar -P /opt/flink/lib/
|
|
|
|
# Avro format
|
|
ARG AVRO_VERSION=2.2.1
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-avro/${AVRO_VERSION}/flink-avro-${AVRO_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# Avro Confluent Registry
|
|
ARG AVRO_REGISTRY_VERSION=2.2.1
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-avro-confluent-registry/${AVRO_REGISTRY_VERSION}/flink-avro-confluent-registry-${AVRO_REGISTRY_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# ---- Version-independent connectors ----
|
|
|
|
# ClickHouse SQL connector
|
|
RUN wget -q https://repo1.maven.org/maven2/name/nkonev/flink/flink-sql-connector-clickhouse/1.17.1-8/flink-sql-connector-clickhouse-1.17.1-8.jar -P /opt/flink/lib/
|
|
|
|
# PostgreSQL CDC
|
|
ARG PG_CDC_VERSION=3.6.0-2.2
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-sql-connector-postgres-cdc/${PG_CDC_VERSION}/flink-sql-connector-postgres-cdc-${PG_CDC_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# Legacy Avro
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/avro/avro/1.8.2/avro-1.8.2.jar -P /opt/flink/lib/
|
|
|
|
# misc libs
|
|
RUN wget -q https://repo1.maven.org/maven2/net/objecthunter/exp4j/0.4.5/exp4j-0.4.5.jar -P /opt/flink/lib/
|
|
RUN wget -q https://jdbc.postgresql.org/download/postgresql-42.7.4.jar -P /opt/flink/lib/
|
|
|
|
# JDBC driver
|
|
ARG JDBC_DRIVER_VERSION=2.2.1
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-sql-jdbc-driver/${JDBC_DRIVER_VERSION}/flink-sql-jdbc-driver-${JDBC_DRIVER_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# Legacy Scala JDBC connector (no 2.x version exists)
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-jdbc_2.12/1.10.3/flink-jdbc_2.12-1.10.3.jar -P /opt/flink/lib/
|
|
|
|
# JDBC connector (latest: 3.3.0-1.20, no 2.x version yet)
|
|
ARG JDBC_CONNECTOR_VERSION=3.3.0-1.20
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-connector-jdbc/${JDBC_CONNECTOR_VERSION}/flink-connector-jdbc-${JDBC_CONNECTOR_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# misc libs
|
|
RUN wget -q https://repo1.maven.org/maven2/com/aventrix/jnanoid/jnanoid/2.0.0/jnanoid-2.0.0.jar -P /opt/flink/lib/
|
|
|
|
# S3 filesystem
|
|
ARG S3_FS_VERSION=2.2.1
|
|
RUN wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-presto/${S3_FS_VERSION}/flink-s3-fs-presto-${S3_FS_VERSION}.jar -P /opt/flink/lib/
|
|
|
|
# Compression
|
|
RUN wget -q https://repo1.maven.org/maven2/com/github/luben/zstd-jni/1.5.7-2/zstd-jni-1.5.7-2.jar -P /opt/flink/lib/
|
|
|
|
# Command to start Flink JobManager and TaskManager in a mini-cluster setup
|
|
CMD ["bin/start-cluster.sh"] |