Skip to content

Commit 85e8b44

Browse files
YC27YC27
authored
Collector: Add collector-distribution packaging module & Migrate IoTDBDataRegionSyncConnector & Add scheduled triggering for batch accumulation operations & Add subscribe source demo & Update iotdb dependence version (apache#61)
1 parent 939ee61 commit 85e8b44

File tree

125 files changed

+11860
-179
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

125 files changed

+11860
-179
lines changed

‎.github/workflows/code-analysis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
uses: github/codeql-action/init@v3
3535
with:
3636
languages: ${{ matrix.language }}
37-
queries: +security-extended
37+
queries: +security-and-quality
3838

3939
- name: Auto Build
4040
uses: github/codeql-action/autobuild@v3

‎.github/workflows/compile-check.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@ jobs:
4747
- name: Compiler Test
4848
shell: bash
4949
run: |
50-
mvn clean compile -P with-integration-tests -ntp
50+
mvn clean package -P with-integration-tests -ntp

‎iotdb-collector/collector-core/pom.xml‎

Lines changed: 136 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,20 @@
5050
</dependency>
5151
<dependency>
5252
<groupId>org.apache.iotdb</groupId>
53-
<artifactId>node-commons</artifactId>
54-
<version>1.3.2</version>
53+
<artifactId>service-rpc</artifactId>
5554
</dependency>
5655
<dependency>
5756
<groupId>org.apache.iotdb</groupId>
58-
<artifactId>service-rpc</artifactId>
59-
<version>1.3.2</version>
57+
<artifactId>pipe-api</artifactId>
58+
</dependency>
59+
<dependency>
60+
<groupId>org.apache.tsfile</groupId>
61+
<artifactId>tsfile</artifactId>
62+
<version>2.1.0-250325-SNAPSHOT</version>
6063
</dependency>
6164
<dependency>
6265
<groupId>org.apache.iotdb</groupId>
63-
<artifactId>pipe-api</artifactId>
66+
<artifactId>tsfile</artifactId>
6467
<version>1.3.2</version>
6568
</dependency>
6669
<dependency>
@@ -103,16 +106,110 @@
103106
<groupId>com.lmax</groupId>
104107
<artifactId>disruptor</artifactId>
105108
</dependency>
109+
<dependency>
110+
<groupId>org.java-websocket</groupId>
111+
<artifactId>Java-WebSocket</artifactId>
112+
</dependency>
113+
<dependency>
114+
<groupId>org.apache.commons</groupId>
115+
<artifactId>commons-lang3</artifactId>
116+
</dependency>
117+
<dependency>
118+
<groupId>org.apache.commons</groupId>
119+
<artifactId>commons-collections4</artifactId>
120+
</dependency>
121+
<dependency>
122+
<groupId>org.apache.commons</groupId>
123+
<artifactId>commons-pool2</artifactId>
124+
</dependency>
125+
<dependency>
126+
<groupId>org.apache.iotdb</groupId>
127+
<artifactId>iotdb-thrift-commons</artifactId>
128+
</dependency>
129+
<dependency>
130+
<groupId>org.apache.iotdb</groupId>
131+
<artifactId>iotdb-thrift</artifactId>
132+
</dependency>
133+
<dependency>
134+
<groupId>org.apache.iotdb</groupId>
135+
<artifactId>iotdb-session</artifactId>
136+
</dependency>
137+
<dependency>
138+
<groupId>net.java.dev.jna</groupId>
139+
<artifactId>jna</artifactId>
140+
</dependency>
141+
<dependency>
142+
<groupId>net.java.dev.jna</groupId>
143+
<artifactId>jna-platform</artifactId>
144+
</dependency>
145+
<dependency>
146+
<groupId>org.apache.thrift</groupId>
147+
<artifactId>libthrift</artifactId>
148+
</dependency>
149+
<dependency>
150+
<groupId>com.google.guava</groupId>
151+
<artifactId>guava</artifactId>
152+
</dependency>
153+
<dependency>
154+
<groupId>org.eclipse.milo</groupId>
155+
<artifactId>stack-core</artifactId>
156+
</dependency>
157+
<dependency>
158+
<groupId>org.eclipse.milo</groupId>
159+
<artifactId>sdk-core</artifactId>
160+
</dependency>
161+
<dependency>
162+
<groupId>org.eclipse.milo</groupId>
163+
<artifactId>stack-server</artifactId>
164+
</dependency>
165+
<dependency>
166+
<groupId>org.eclipse.milo</groupId>
167+
<artifactId>sdk-server</artifactId>
168+
</dependency>
169+
<dependency>
170+
<groupId>org.apache.tsfile</groupId>
171+
<artifactId>common</artifactId>
172+
</dependency>
173+
<dependency>
174+
<groupId>io.moquette</groupId>
175+
<artifactId>moquette-broker</artifactId>
176+
</dependency>
106177
<dependency>
107178
<groupId>com.google.code.findbugs</groupId>
108179
<artifactId>jsr305</artifactId>
109180
</dependency>
181+
<dependency>
182+
<groupId>ch.qos.logback</groupId>
183+
<artifactId>logback-classic</artifactId>
184+
</dependency>
110185
<dependency>
111186
<groupId>org.xerial</groupId>
112187
<artifactId>sqlite-jdbc</artifactId>
113188
</dependency>
189+
<dependency>
190+
<groupId>commons-io</groupId>
191+
<artifactId>commons-io</artifactId>
192+
</dependency>
193+
<dependency>
194+
<groupId>com.github.luben</groupId>
195+
<artifactId>zstd-jni</artifactId>
196+
</dependency>
197+
<dependency>
198+
<groupId>com.github.ben-manes.caffeine</groupId>
199+
<artifactId>caffeine</artifactId>
200+
</dependency>
114201
</dependencies>
115202
<build>
203+
<resources>
204+
<resource>
205+
<directory>${project.basedir}/src/assembly/resources/conf</directory>
206+
<includes>
207+
<include>application.properties</include>
208+
<include>logback.xml</include>
209+
</includes>
210+
<filtering>false</filtering>
211+
</resource>
212+
</resources>
116213
<plugins>
117214
<plugin>
118215
<groupId>org.apache.maven.plugins</groupId>
@@ -126,6 +223,40 @@
126223
</usedDependencies>
127224
</configuration>
128225
</plugin>
226+
<plugin>
227+
<groupId>org.apache.maven.plugins</groupId>
228+
<artifactId>maven-compiler-plugin</artifactId>
229+
<configuration>
230+
<source>8</source>
231+
<target>8</target>
232+
</configuration>
233+
</plugin>
234+
<plugin>
235+
<groupId>org.apache.maven.plugins</groupId>
236+
<artifactId>maven-assembly-plugin</artifactId>
237+
<executions>
238+
<!-- Package binaries-->
239+
<execution>
240+
<id>server-assembly</id>
241+
<goals>
242+
<goal>single</goal>
243+
</goals>
244+
<phase>package</phase>
245+
<configuration>
246+
<descriptors>
247+
<descriptor>src/assembly/core.xml</descriptor>
248+
</descriptors>
249+
<appendAssemblyId>false</appendAssemblyId>
250+
<archive>
251+
<manifest>
252+
<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
253+
<addDefaultSpecificationEntries>true</addDefaultSpecificationEntries>
254+
</manifest>
255+
</archive>
256+
</configuration>
257+
</execution>
258+
</executions>
259+
</plugin>
129260
</plugins>
130261
</build>
131262
</project>

‎iotdb-collector/collector-core/src/assembly/core.xml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
4+
Licensed to the Apache Software Foundation (ASF) under one
5+
or more contributor license agreements. See the NOTICE file
6+
distributed with this work for additional information
7+
regarding copyright ownership. The ASF licenses this file
8+
to you under the Apache License, Version 2.0 (the
9+
"License"); you may not use this file except in compliance
10+
with the License. You may obtain a copy of the License at
11+
12+
http://www.apache.org/licenses/LICENSE-2.0
13+
14+
Unless required by applicable law or agreed to in writing,
15+
software distributed under the License is distributed on an
16+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
KIND, either express or implied. See the License for the
18+
specific language governing permissions and limitations
19+
under the License.
20+
21+
-->
22+
<assembly>
23+
<id>server</id>
24+
<formats>
25+
<format>dir</format>
26+
<format>zip</format>
27+
</formats>
28+
<includeBaseDirectory>false</includeBaseDirectory>
29+
<dependencySets>
30+
<dependencySet>
31+
<outputDirectory>lib</outputDirectory>
32+
</dependencySet>
33+
</dependencySets>
34+
<fileSets>
35+
<fileSet>
36+
<directory>src/assembly/resources</directory>
37+
<outputDirectory>${file.separator}</outputDirectory>
38+
</fileSet>
39+
</fileSets>
40+
</assembly>

‎iotdb-collector/collector-core/src/assembly/resources/conf/application.properties‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
####################
21+
### API Service Configuration
22+
####################
23+
24+
# The port on which the Jetty service runs.
25+
# Effective mode: on every start
26+
# Data type: int
27+
api_service_port=17070
28+
29+
####################
30+
### Task Runtime Configuration
31+
####################
32+
33+
# The number of concurrent threads for the source task.
34+
# Effective mode: on every start
35+
# Data type: int
36+
task_source_parallelism_num=1
37+
38+
# The number of concurrent threads for the process task.
39+
# Effective mode: on every start
40+
# Data type: int
41+
task_process_parallelism_num=4
42+
43+
# The number of concurrent threads for the sink task.
44+
# Effective mode: on every start
45+
# Data type: int
46+
task_sink_parallelism_num=4
47+
48+
# The ring buffer size for the processor task.
49+
# Effective mode: on every start
50+
# Data type: int
51+
task_processor_ring_buffer_size=1024
52+
53+
# The ring buffer size for the sink task.
54+
# Effective mode: on every start
55+
# Data type: int
56+
task_sink_ring_buffer_size=1024
57+
58+
# Database file location of task
59+
# Effective mode: on every start
60+
# Data type: string
61+
task_database_file_path=system/database/task.db
62+
63+
####################
64+
### Plugin Configuration
65+
####################
66+
67+
# The location of plugin jar file
68+
# Effective mode: on every start
69+
# Data type: string
70+
plugin_lib_dir=system/plugin
71+
72+
# Installation location of plugin jar file
73+
# Effective mode: on every start
74+
# Data type: string
75+
plugin_install_lib_dir=system/plugin/install
76+
77+
# Database file location of plugin
78+
# Effective mode: on every start
79+
# Data type: string
80+
plugin_database_file_path=system/database/plugin.db
81+
82+
####################
83+
### Pipe Configuration
84+
####################
85+
86+
# The total bytes that all pipe sinks can transfer per second.
87+
# When given a value less than or equal to 0, it means no limit.
88+
# default value is -1, which means no limit.
89+
# Effective mode: on every start
90+
# Data type: double
91+
pipe_all_sinks_rate_limit_bytes_per_second=-1
92+
93+
# Rate limiter configuration interval in milliseconds for hot reloading
94+
# Effective mode: on every start
95+
# Data type: int
96+
rate_limiter_hot_reload_check_interval_ms=1000
97+
98+
# Maximum number of retry attempts for operations
99+
# Effective mode: on every start
100+
# Data type: int
101+
max_retry_times=5
102+
103+
# Used for connection of IoTDB native clients
104+
# Bind with rpc_address
105+
# Effective mode: on every start
106+
# Data type: int
107+
rpc_port=6667
108+
109+
# Used for connection of IoTDB native clients(Session)
110+
# Could set 127.0.0.1(for local test) or ipv4 address
111+
# Effective mode: on every start
112+
# Data type: String
113+
rpc_address=0.0.0.0
114+
115+
# Buffer size for reading files in pipe connector (8MB default)
116+
# Effective mode: on every start
117+
# Data type: int
118+
pipe_connector_read_file_buffer_size=8388608
119+
120+
# Timeout duration for pipe connector data transfer in milliseconds
121+
# Effective mode: on every start
122+
# Data type: int
123+
pipe_connector_transfer_timeout_ms=900000
124+
125+
# Maximum allowed frame size for Thrift communication
126+
# Effective mode: on every start
127+
# Data type: int
128+
thrift_frame_max_size=536870912
129+
130+
# Enable/disable thrift compression for pipe connector RPC
131+
# Effective mode: on every start
132+
# Data type: boolean
133+
is_pipe_connector_rpc_thrift_compression_enabled=false
134+
135+
# Use this value to set timestamp precision as "ms", "us" or "ns".
136+
# Once the precision has been set, it can not be changed.
137+
# Effective mode: on every start
138+
# Data type: string
139+
timestamp_precision=ms
140+
141+
# Memory allocation ratio for pipe leader cache management
142+
# Effective mode: on every start
143+
# Data type: float
144+
pipe_leader_cache_memory_usage_percentage=0.1
145+
146+
# Enable/disable reference tracking for pipe events
147+
# Effective mode: on every start
148+
# Data type: boolean
149+
pipe_event_reference_tracking_enabled=true
150+
151+
# Proactively triggers the interval for batch deliveries
152+
# Effective mode: on every start
153+
# Data type: long
154+
executor_cron_heartbeat_event_interval_seconds=20

0 commit comments

Comments
 (0)