Skip to content

Commit 016f6b8

Browse files
zhisheng17zhisheng17
committed
add flink sql client, can run flink sql job
1 parent d6d817b commit 016f6b8

27 files changed

+1354
-0
lines changed

‎flink-learning-sql/flink-learning-sql-client/README.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
2+
3+
k8s application 任务提交命令
4+
5+
```shell
6+
/app/flink-1.16.1-sql/bin/flink run-application -t kubernetes-application -Dkubernetes.cluster-id=batch-flink-373362-1741687273399 -Dexecution.job-listeners=com.zhisheng.plugins.jobStatusListener -Djob.alert.ownerList=xxx -Denv.java.opts.taskmanager="-DtaskName=1618789_1739795455429 -DtaskId=373362" -Denv.java.opts.jobmanager="-DtaskName=1618789_1739795455429 -DtaskId=373362" -Denv.java.opts.client="-DtaskId=373362" -Dkubernetes.container.image=harbor.xxx/bigdata/links_sql_116_common_image_pro:2024122414 -Dkubernetes.config.file=/app/zhisheng/k8s/config_idc2 -Dkubernetes.namespace=dataxxx -Dcontainerized.master.env.LINKS_JOB_FILE=oss://xxx/release/373362.sql -Dcontainerized.master.env.relayJarsOssPath=oss:///functions/FlinkUDF-flink-dataman-udf.jar,oss://prod/external/iceberg_flink1.16/iceberg-flink-runtime-1.16-1.4.0-tag-hellobike-20231027.jar,oss://functions/celeborn-client-flink-1.16-shaded_2.12-0.5.0.jar -Dcontainerized.master.env.LINKS_USER_NAME=xxx_deploy -Dcontainerized.taskmanager.env.LINKS_JOB_FILE=oss://sql/xxx_deploy/release/373362.sql -Dcontainerized.taskmanager.env.relayJarsOssPath=oss://functions/FlinkUDF-flink-dataman-udf.jar,oss://external/iceberg_flink1.16/iceberg-flink-runtime-1.16-1.4.0-tag-hellobike-20231027.jar,oss://functions/celeborn-client-flink-1.16-shaded_2.12-0.5.0.jar -Dcontainerized.taskmanager.env.LINKS_USER_NAME=xxx_deploy -Dtaskmanager.numberOfTaskSlots=1 -Djobmanager.memory.process.size=2048m -Dtaskmanager.memory.process.size=3072m -Dparallelism.default=1 local:///opt/flink/usrlib/links-sql.jar -w /opt/flink/usrlib -f 373362.sql -t false -k8d batch-flink-373362-1741687273399 -b true
7+
```
8+
9+
注意：
10+
11+
`-w /opt/flink/usrlib -f 373362.sql -t false -k8d batch-flink-373362-1741687273399 -b true`
12+
13+
这段内容是 Flink SQL 命令，其中 `-f 373362.sql` 是指 SQL 文件的路径，`-t false` 是指是否是临时任务，`-k8d batch-flink-373362-1741687273399` 是指 Flink 的 Appliation ID，`-b true` 是指是否是 Batch 任务。

‎flink-learning-sql/flink-learning-sql-client/pom.xml

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3+
<parent>
4+
<artifactId>flink-learning-sql</artifactId>
5+
<groupId>com.zhisheng.flink</groupId>
6+
<version>1.0-SNAPSHOT</version>
7+
</parent>
8+
<modelVersion>4.0.0</modelVersion>
9+
<artifactId>flink-learning-sql-client</artifactId>
10+
<name>Archetype - flink-learning-sql-client</name>
11+
12+
<properties>
13+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
14+
</properties>
15+
16+
<dependencies>
17+
<dependency>
18+
<groupId>com.alibaba</groupId>
19+
<artifactId>fastjson</artifactId>
20+
<version>1.2.54</version>
21+
</dependency>
22+
<dependency>
23+
<groupId>org.antlr</groupId>
24+
<artifactId>antlr4</artifactId>
25+
<version>4.7.2</version>
26+
</dependency>
27+
<dependency>
28+
<groupId>org.apache.flink</groupId>
29+
<artifactId>flink-table-common</artifactId>
30+
<version>${flink.version}</version>
31+
<scope>provided</scope>
32+
</dependency>
33+
<dependency>
34+
<groupId>org.apache.flink</groupId>
35+
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
36+
<version>${flink.version}</version>
37+
<scope>provided</scope>
38+
</dependency>
39+
40+
<dependency>
41+
<groupId>org.apache.flink</groupId>
42+
<artifactId>flink-table-api-java</artifactId>
43+
<version>${flink.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
47+
<dependency>
48+
<groupId>org.apache.flink</groupId>
49+
<artifactId>flink-table-planner_${scala.binary.version}</artifactId>
50+
<version>${flink.version}</version>
51+
<scope>provided</scope>
52+
</dependency>
53+
54+
<dependency>
55+
<groupId>org.apache.flink</groupId>
56+
<artifactId>flink-clients_${scala.binary.version}</artifactId>
57+
<version>${flink.version}</version>
58+
<scope>provided</scope>
59+
</dependency>
60+
61+
<dependency>
62+
<groupId>org.apache.flink</groupId>
63+
<artifactId>flink-java</artifactId>
64+
<version>${flink.version}</version>
65+
<scope>provided</scope>
66+
</dependency>
67+
<dependency>
68+
<groupId>org.apache.flink</groupId>
69+
<artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
70+
<version>${flink.version}</version>
71+
<scope>provided</scope>
72+
</dependency>
73+
<dependency>
74+
<groupId>org.apache.flink</groupId>
75+
<artifactId>flink-json</artifactId>
76+
<version>${flink.version}</version>
77+
<scope>provided</scope>
78+
</dependency>
79+
80+
<dependency>
81+
<groupId>org.apache.flink</groupId>
82+
<artifactId>flink-sql-orc_${scala.binary.version}</artifactId>
83+
<version>${flink.version}</version>
84+
<scope>provided</scope>
85+
</dependency>
86+
87+
<dependency>
88+
<groupId>org.apache.flink</groupId>
89+
<artifactId>flink-sql-avro</artifactId>
90+
<version>${flink.version}</version>
91+
<scope>provided</scope>
92+
</dependency>
93+
94+
<dependency>
95+
<groupId>org.apache.flink</groupId>
96+
<artifactId>flink-sql-parquet_${scala.binary.version}</artifactId>
97+
<version>${flink.version}</version>
98+
<scope>provided</scope>
99+
</dependency>
100+
<dependency>
101+
<groupId>junit</groupId>
102+
<artifactId>junit</artifactId>
103+
<version>4.13.1</version>
104+
<scope>test</scope>
105+
</dependency>
106+
<dependency>
107+
<groupId>mysql</groupId>
108+
<artifactId>mysql-connector-java</artifactId>
109+
<version>5.1.44</version>
110+
<scope>provided</scope>
111+
</dependency>
112+
<dependency>
113+
<groupId>org.postgresql</groupId>
114+
<artifactId>postgresql</artifactId>
115+
<version>42.2.8</version>
116+
<scope>provided</scope>
117+
</dependency>
118+
<dependency>
119+
<groupId>org.apache.flink</groupId>
120+
<artifactId>flink-sql-connector-elasticsearch7_${scala.binary.version}</artifactId>
121+
<version>${flink.version}</version>
122+
<scope>provided</scope>
123+
</dependency>
124+
<dependency>
125+
<groupId>org.apache.flink</groupId>
126+
<artifactId>flink-sql-connector-kafka_${scala.binary.version}</artifactId>
127+
<version>${flink.version}</version>
128+
<scope>provided</scope>
129+
</dependency>
130+
131+
<dependency>
132+
<groupId>org.apache.flink</groupId>
133+
<artifactId>flink-sql-connector-hbase-2.2_${scala.binary.version}</artifactId>
134+
<version>${flink.version}</version>
135+
<scope>provided</scope>
136+
</dependency>
137+
138+
<dependency>
139+
<groupId>org.apache.flink</groupId>
140+
<artifactId>flink-sql-connector-hive-2.2.0_${scala.binary.version}</artifactId>
141+
<version>${flink.version}</version>
142+
<scope>provided</scope>
143+
</dependency>
144+
</dependencies>
145+
146+
<profiles>
147+
<profile>
148+
<id>dev</id>
149+
<activation>
150+
<activeByDefault>true</activeByDefault>
151+
</activation>
152+
<build>
153+
<resources>
154+
<resource>
155+
<directory>src/main/resources/dev</directory>
156+
<includes>
157+
<include>**/*</include>
158+
</includes>
159+
<filtering>true</filtering>
160+
</resource>
161+
</resources>
162+
</build>
163+
</profile>
164+
<profile>
165+
<id>fat</id>
166+
<build>
167+
<resources>
168+
<resource>
169+
<directory>src/main/resources/fat</directory>
170+
<includes>
171+
<include>**/*</include>
172+
</includes>
173+
<filtering>true</filtering>
174+
</resource>
175+
</resources>
176+
</build>
177+
</profile>
178+
<profile>
179+
<id>prod</id>
180+
<build>
181+
<resources>
182+
<resource>
183+
<directory>src/main/resources/prod</directory>
184+
<includes>
185+
<include>**/*</include>
186+
</includes>
187+
<filtering>true</filtering>
188+
</resource>
189+
</resources>
190+
</build>
191+
</profile>
192+
</profiles>
193+
194+
<build>
195+
<plugins>
196+
<plugin>
197+
<groupId>org.apache.maven.plugins</groupId>
198+
<artifactId>maven-compiler-plugin</artifactId>
199+
<version>3.6.0</version>
200+
<configuration>
201+
<source>1.8</source>
202+
<target>1.8</target>
203+
</configuration>
204+
</plugin>
205+
<plugin>
206+
<groupId>org.apache.maven.plugins</groupId>
207+
<artifactId>maven-shade-plugin</artifactId>
208+
<executions>
209+
<execution>
210+
<id>zhisheng-sql</id>
211+
<phase>package</phase>
212+
<goals>
213+
<goal>shade</goal>
214+
</goals>
215+
<configuration>
216+
<finalName>${project.name}</finalName>
217+
<transformers>
218+
<transformer
219+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
220+
<mainClass>com.zhisheng.sql.SqlSubmit</mainClass>
221+
</transformer>
222+
<transformer
223+
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
224+
<resource>META-INF/services/org.apache.flink.table.factories.TableFactory</resource>
225+
</transformer>
226+
</transformers>
227+
</configuration>
228+
</execution>
229+
</executions>
230+
</plugin>
231+
232+
<plugin>
233+
<artifactId>maven-assembly-plugin</artifactId>
234+
<configuration>
235+
<archive>
236+
<manifest>
237+
<mainClass>com.zhisheng.sql.SqlSubmit</mainClass>
238+
</manifest>
239+
<manifestEntries>
240+
<Class-Path>.</Class-Path>
241+
</manifestEntries>
242+
</archive>
243+
<finalName>${project.name}</finalName>
244+
<descriptorRefs>
245+
<descriptorRef>jar-with-dependencies</descriptorRef>
246+
</descriptorRefs>
247+
</configuration>
248+
<executions>
249+
<execution>
250+
<id>make-assembly</id>
251+
<phase>package</phase>
252+
<goals>
253+
<goal>single</goal>
254+
</goals>
255+
</execution>
256+
</executions>
257+
</plugin>
258+
</plugins>
259+
</build>
260+
</project>

‎flink-learning-sql/flink-learning-sql-client/src/main/java/com/zhisheng/sql/SqlSubmit.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.zhisheng.sql;
2+
3+
4+
import com.zhisheng.sql.cli.CliOptions;
5+
import com.zhisheng.sql.cli.CliOptionsParser;
6+
import com.zhisheng.sql.planner.BatchPlanner;
7+
import com.zhisheng.sql.planner.Planner;
8+
import com.zhisheng.sql.planner.StreamingPlanner;
9+
10+
public class SqlSubmit {
11+
12+
public static void main(String[] args) throws Exception {
13+
14+
final CliOptions options = CliOptionsParser.parseClient(args);
15+
Planner planner;
16+
if (!Boolean.parseBoolean(options.getIsBatch())) {
17+
planner = StreamingPlanner.build(options);
18+
} else {
19+
planner = BatchPlanner.build(options);
20+
}
21+
22+
planner.run();
23+
}
24+
}

‎flink-learning-sql/flink-learning-sql-client/src/main/java/com/zhisheng/sql/cli/CliOptions.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package com.zhisheng.sql.cli;
2+
3+
4+
public class CliOptions {
5+
6+
private final String sqlFilePath;
7+
private final String workingSpace;
8+
private final String isTest;
9+
private final String isBatch;
10+
private final String k8sClusterId;
11+
12+
public CliOptions(String sqlFilePath, String workingSpace, String isTest, String isBatch, String k8sClusterId) {
13+
this.sqlFilePath = sqlFilePath;
14+
this.workingSpace = workingSpace;
15+
this.isTest = isTest;
16+
this.isBatch = isBatch;
17+
this.k8sClusterId = k8sClusterId;
18+
}
19+
20+
public String getSqlFilePath() {
21+
return sqlFilePath;
22+
}
23+
24+
public String getWorkingSpace() {
25+
return workingSpace;
26+
}
27+
28+
public String getIsTest() {
29+
return isTest;
30+
}
31+
32+
public String getIsBatch() {
33+
return isBatch;
34+
}
35+
36+
public String getK8sClusterId() {
37+
return k8sClusterId;
38+
}
39+
}

0 commit comments

Comments
 (0)