Skip to content

Commit

Permalink
Merge branch 'apache:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
omkar-shitole authored Dec 31, 2024
2 parents f170f5e + 6ecd873 commit ce175f9
Show file tree
Hide file tree
Showing 1,033 changed files with 14,672 additions and 5,498 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/nightly-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: Nightly - Build

on:
schedule:
- cron: '0 16 */1 * *' # once a day. UTC time
- cron: '0 16 * * *' # Once a day. UTC time
workflow_dispatch:
inputs:
commit-id:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/nightly-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: Nightly - Check

on:
schedule:
- cron: '0 23 */1 * *' # once a day. UTC time
- cron: '0 23 * * 1-5' # Once a day between Monday and Friday. UTC time
workflow_dispatch:

env:
Expand Down Expand Up @@ -126,7 +126,7 @@ jobs:
days-before-pr-stale: -1
days-before-close: -1
stale-issue-label: "stale"
any-of-issue-labels: "status: need more info,type: question,type: duplicate"
any-of-issue-labels: "status: need more info,type: question,type: duplicate,status: invalid"
operations-per-run: 100
remove-stale-when-updated: true
stale-issue-message: >
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: Nightly - CI

on:
schedule:
- cron: '0 13 */1 * *' # UTC time
- cron: '0 13 * * 1-5' # Once a day between Monday and Friday. UTC time
workflow_dispatch:

env:
Expand Down
145 changes: 145 additions & 0 deletions .github/workflows/nightly-e2e-agent.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name: Nightly - E2E Agent

on:
schedule:
- cron: '0 0 * * 1-5' # Once a day between Monday and Friday. UTC time
workflow_dispatch:

concurrency:
group: e2e-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true

jobs:
global-environment:
name: Import Global Environment
uses: ./.github/workflows/required-reusable.yml

build-e2e-image:
if: github.repository == 'apache/shardingsphere'
name: Build E2E Image
needs: global-environment
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 21
- uses: actions/cache@v4
with:
path: ~/.m2/repository
key: apache-shardingsphere-maven-third-party-e2e-cache-${{ github.sha }}
restore-keys: |
apache-shardingsphere-maven-third-party-e2e-cache-
apache-shardingsphere-maven-third-party-
- name: Build Project
run: ./mvnw -B clean install -DskipTests -Prelease
- name: Build Proxy E2E Image
run: ./mvnw clean install -DskipTests -Pit.env.proxy -f test/e2e/agent/engine/pom.xml
- name: Save Proxy E2E Image
run: docker save -o /tmp/apache-shardingsphere-proxy-agent-test.tar apache/shardingsphere-proxy-agent-test:latest
- uses: actions/upload-artifact@v4
with:
name: proxy-e2e-image
path: /tmp/apache-shardingsphere-proxy-agent-test.tar
retention-days: 10
- name: Build JDBC E2E Image
run: ./mvnw clean install -DskipTests -Pit.env.jdbc -f test/e2e/agent/engine/pom.xml
- name: Save JDBC E2E Image
run: docker save -o /tmp/apache-shardingsphere-jdbc-agent-test.tar apache/shardingsphere-jdbc-agent-test:latest
- uses: actions/upload-artifact@v4
with:
name: jdbc-e2e-image
path: /tmp/apache-shardingsphere-jdbc-agent-test.tar
retention-days: 10

agent-mysql:
name: E2E - Agent with MySQL
if: github.repository == 'apache/shardingsphere'
needs: [ global-environment, build-e2e-image ]
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
max-parallel: 20
fail-fast: false
matrix:
include:
- adapter: proxy
feature: tracing
plugin: jaeger
- adapter: proxy
feature: tracing
plugin: zipkin
- adapter: proxy
feature: metrics
plugin: prometheus
- adapter: proxy
feature: logging
plugin: file
- adapter: jdbc
feature: logging
plugin: file
- adapter: jdbc
feature: metrics
plugin: prometheus
- adapter: jdbc
feature: tracing
plugin: jaeger
- adapter: jdbc
feature: tracing
plugin: zipkin
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: 21
- uses: actions/cache@v4
with:
path: ~/.m2/repository
key: apache-shardingsphere-maven-third-party-${{ hashFiles('**/pom.xml') }}
restore-keys: |
apache-shardingsphere-maven-third-party-
- name: Download Proxy E2E Image
if: matrix.adapter == 'proxy'
uses: actions/download-artifact@v4
with:
name: proxy-e2e-image
path: /tmp/
- name: Load Proxy E2E Image
if: matrix.adapter == 'proxy'
run: docker load -i /tmp/apache-shardingsphere-proxy-agent-test.tar
- name: Download JDBC E2E Image
if: matrix.adapter == 'jdbc'
uses: actions/download-artifact@v4
with:
name: jdbc-e2e-image
path: /tmp/
- name: Load JDBC E2E Image
if: matrix.adapter == 'jdbc'
run: docker load -i /tmp/apache-shardingsphere-jdbc-agent-test.tar
- name: Build Project
run: ./mvnw -B clean install -am -pl test/e2e/agent/plugins/${{ matrix.feature }}/${{ matrix.plugin }} -DskipTests
- name: Run E2E Test
run: ./mvnw -nsu -B clean install -f test/e2e/agent/plugins/${{ matrix.feature }}/${{ matrix.plugin }}/pom.xml -Dit.env.adapter=${{ matrix.adapter }} -Dit.env.plugin=${{ matrix.plugin }}
2 changes: 1 addition & 1 deletion .github/workflows/nightly-e2e-operation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: Nightly - E2E Operation

on:
schedule:
- cron: '0 20 */1 * *' # UTC time
- cron: '0 19 * * 1-5' # Once a day between Monday and Friday. UTC time
workflow_dispatch:

concurrency:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly-e2e-sql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: Nightly - E2E SQL

on:
schedule:
- cron: '0 15 */1 * *' # once a day. UTC time
- cron: '0 15 * * 1-5' # Once a day between Monday and Friday. UTC time
workflow_dispatch:

concurrency:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/nightly-sql-parser.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ name: Nightly - SQL parser loading from external cases

on:
schedule:
- cron: '0 18 */1 * *' # once a day. UTC time
- cron: '0 18 * * 1-5' # Once a day between Monday and Friday. UTC time
workflow_dispatch:

env:
Expand Down
36 changes: 30 additions & 6 deletions RELEASE-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,44 +11,67 @@
1. Kernel: Add arguments not null check when creating RouteUnit - [#33382](https://github.com/apache/shardingsphere/pull/33382)
1. Kernel: Add index columns not empty judgement for IndexColumnTokenGenerator - [#33384](https://github.com/apache/shardingsphere/pull/33384)
1. Kernel: Add binding to owner table - [#33533](https://github.com/apache/shardingsphere/pull/33533)
1. Kernel: Add binding to owner table - [#33533](https://github.com/apache/shardingsphere/pull/33533)
1. Kernel: Add WithAvailable interface and encrypt with, combine, insert select support checker - [#34175](https://github.com/apache/shardingsphere/pull/34175)
1. Metadata: Add load-table-metadata-batch-size props to concurrent load table metadata - [#34009](https://github.com/apache/shardingsphere/pull/34009)
1. DistSQL: Check inline expression when create sharding table rule with inline sharding algorithm - [#33735](https://github.com/apache/shardingsphere/pull/33735)
1. SQL Parser: Support parsing Doris BITXOR - [#33258](https://github.com/apache/shardingsphere/pull/33258)
1. SQL Parser: Support parsing Doris INSTR - [#33289](https://github.com/apache/shardingsphere/pull/33289)
1. SQL Parser: Support parsing Doris STRRIGHT - [#33393](https://github.com/apache/shardingsphere/pull/33393)
1. SQL Parser: Support parsing MySQL by adding non-reserved keywords in BaseRule.g4 file according to MySQL 8.4 doc - [#33846](https://github.com/apache/shardingsphere/pull/33846)
1. SQL Parser: Support parsing Doris EXTRACT\_URL\_PARAMETER - [#33571](https://github.com/apache/shardingsphere/pull/33571)
1. SQL Binder: Add sql bind logic for create table statement - [#34074](https://github.com/apache/shardingsphere/pull/34074)
1. SQL Binder: Support create index statement sql bind - [#34112](https://github.com/apache/shardingsphere/pull/34112)
1. SQL Parser: Support MySQL update with statement parse - [#34126](https://github.com/apache/shardingsphere/pull/34126)
1. SQL Binder: Remove TablesContext#findTableNames method and implement select order by, group by bind logic - [#34123](https://github.com/apache/shardingsphere/pull/34123)
1. SQL Binder: Support select with statement sql bind and add bind test case - [#34141](https://github.com/apache/shardingsphere/pull/34141)
1. SQL Binder: Support sql bind for select with current select projection reference - [#34151](https://github.com/apache/shardingsphere/pull/34151)
1. SQL Binder: Support alter table, drop table sql bind and add test case - [#34154](https://github.com/apache/shardingsphere/pull/34154)
1. SQL Binder: Support rename table statement sql bind and split segment bind to ddl and dml package - [#34158](https://github.com/apache/shardingsphere/pull/34158)
1. SQL Binder: Support copy statement sql bind and add bind test case - [#34159](https://github.com/apache/shardingsphere/pull/34159)
1. SQL Binder: Support truncate table sql bind and add test case - [#34162](https://github.com/apache/shardingsphere/pull/34162)
1. SQL Binder: Support create view, alter view, drop view sql bind logic - [#34167](https://github.com/apache/shardingsphere/pull/34167)
1. SQL Binder: Support load data and load xml sql bind and add test case - [#34177](https://github.com/apache/shardingsphere/pull/34177)
1. Storage: Support setting `hive_conf_list`, `hive_var_list` and `sess_var_list` for jdbcURL when connecting to HiveServer2 - [#33749](https://github.com/apache/shardingsphere/pull/33749)
1. Storage: Support connecting to HiveServer2 through database connection pools other than HikariCP - [#33762](https://github.com/apache/shardingsphere/pull/33762)
1. Storage: Partial support for connecting to embedded ClickHouse `chDB` - [#33786](https://github.com/apache/shardingsphere/pull/33786)
1. SQL Federation: Upgrade calcite version to 1.38.0 and update all license info in LICENSE file - [#33279](https://github.com/apache/shardingsphere/pull/33279)
1. Transaction: Support savepoint/release savepoint TCL statements in jdbc adapter -[#34173](https://github.com/apache/shardingsphere/pull/34173)
1. Transaction: Bump the minimum Seata Client version for Seata AT integration to 2.2.0 - [#33872](https://github.com/apache/shardingsphere/pull/33872)
1. SQL Federation: Upgrade calcite version to 1.38.0 and update all license info in LICENSE file - [#33279](https://github.com/apache/shardingsphere/pull/33279)
1. JDBC: Add show database name for JDBC when execute SHOW COMPUTE NODES - [#33437](https://github.com/apache/shardingsphere/pull/33437)
1. JDBC: Support ZonedDateTime on ResultSet - [#33660](https://github.com/apache/shardingsphere/issues/33660)
1. Proxy: Add query parameters and check for MySQL kill processId - [#33274](https://github.com/apache/shardingsphere/pull/33274)
1. Proxy: Support table not exist exception for PostgreSQL proxy - [#33885](https://github.com/apache/shardingsphere/pull/33274)
1. Proxy Native: Change the Base Docker Image of ShardingSphere Proxy Native - [#33263](https://github.com/apache/shardingsphere/issues/33263)
1. Proxy Native: Support connecting to HiveServer2 with ZooKeeper Service Discovery enabled in GraalVM Native Image - [#33768](https://github.com/apache/shardingsphere/pull/33768)
1. Proxy Native: Support local transactions of ClickHouse under GraalVM Native Image - [#33801](https://github.com/apache/shardingsphere/pull/33801)
1. Sharding: Support MYSQL GroupConcat function for aggregating multiple shards - [#33808](https://github.com/apache/shardingsphere/pull/33808)
1. Proxy Native: Support Seata AT integration under Proxy Native in GraalVM Native Image - [#33889](https://github.com/apache/shardingsphere/pull/33889)
1. Sharding: Support GroupConcat function for aggregating multiple shards in MySQL, OpenGauss, Doris - [#33808](https://github.com/apache/shardingsphere/pull/33808)
1. Agent: Simplify the use of Agent's Docker Image - [#33356](https://github.com/apache/shardingsphere/pull/33356)
1. Mode: Support modifying Hikari-CP configurations via props in standalone mode [#34185](https://github.com/apache/shardingsphere/pull/34185)

### Bug Fixes

1. Mode: Fixes `JDBCRepository` improper handling of H2-database in memory mode - [#33281](https://github.com/apache/shardingsphere/issues/33281)
1. SQL Binder: Fixes table does not exist exception when use HintManager#setDatabaseName to transparent - [#33370](https://github.com/apache/shardingsphere/pull/33370)
1. SQL Parser: Fixes LiteralExpressionSegment cast exception in SQL parser - [#33332](https://github.com/apache/shardingsphere/pull/33332)
1. SQL Parser: Fixes PostgreSQL and openGauss time extract function parse week and quarter error - [#33564](https://github.com/apache/shardingsphere/pull/33564)
1. SQL Parser: Fixes MySQL parse zone unreserved keyword error - [#33720](https://github.com/apache/shardingsphere/pull/33720)
1. SQL Parser: Fixes MySQL range parse error when use table owner - [#33874](https://github.com/apache/shardingsphere/pull/33874)
1. SQL Binder: Fixes table does not exist exception when use HintManager#setDatabaseName to transparent - [#33370](https://github.com/apache/shardingsphere/pull/33370)
1. SQL Binder: Use Multimap and CaseInsensitiveString to replace CaseInsensitiveMap for supporting MySQL multi table join with same table alias - [#33303](https://github.com/apache/shardingsphere/pull/33303)
1. SQL Binder: Fixes the combine statement cannot find the outer table when bind - [#33357](https://github.com/apache/shardingsphere/pull/33357)
1. SQL Binder: Fixes SQL performance issues caused by repeated subquery fetches - [#33361](https://github.com/apache/shardingsphere/pull/33361)
1. SQL Parser: Fixes MySQL parse zone unreserved keyword error - [#33720](https://github.com/apache/shardingsphere/pull/33720)
1. SQL Parser: Fixes MySQL range parse error when use table owner - [#33874](https://github.com/apache/shardingsphere/pull/33874)
1. SQL Binder: Fixes the expression segment cannot find the outer table when binding - [#34015](https://github.com/apache/shardingsphere/pull/34015)
1. Storage: Fixes cannot connect to HiveServer2 using remote Hive Metastore Server - [#33837](https://github.com/apache/shardingsphere/pull/33837)
1. Proxy: Fixes BatchUpdateException when execute INSERT INTO ON DUPLICATE KEY UPDATE in proxy adapter - [#33796](https://github.com/apache/shardingsphere/pull/33796)
1. Proxy: Fixes "ALL PRIVILEGES ON `DB`.*" is not recognized during SELECT privilege verification for MySQL - [#34037](https://github.com/apache/shardingsphere/pull/34037)
1. Proxy: Fixes MySQL longblob wrong column type returned by proxy protocol - [#34121](https://github.com/apache/shardingsphere/pull/34121)
1. Sharding: Remove ShardingRouteAlgorithmException check logic temporarily to support different actual table name configuration - [#33367](https://github.com/apache/shardingsphere/pull/33367)
1. Sharding: Fixes SQL COUNT with GROUP BY to prevent incorrect row returns - [#33380](https://github.com/apache/shardingsphere/pull/33380)
1. Sharding: Fixes avg, sum, min, max function return empty data when no query result return - [#33449](https://github.com/apache/shardingsphere/pull/33449)
1. Encrypt: Fixes merge exception without encrypt rule in database - [#33708](https://github.com/apache/shardingsphere/pull/33708)
1. Encrypt: Use sql bind info in EncryptInsertPredicateColumnTokenGenerator to avoid wrong column table mapping - [#34110](https://github.com/apache/shardingsphere/pull/34110)
1. Mode: Fixes `JDBCRepository` improper handling of H2-database in memory mode - [#33281](https://github.com/apache/shardingsphere/issues/33281)
1. Mode: Fixes duplicate column names added when index changed in DDL - [#33982](https://github.com/apache/shardingsphere/issues/33281)

### Change Logs

Expand Down Expand Up @@ -117,6 +140,7 @@
1. Pipeline: Support page query for inventory dumper and data consistency streaming query
1. Pipeline: Use case-insensitive identifiers to enhance the table metadata loader
1. Pipeline: Support primary key columns ordering for standard pipeline table metadata loader
1. Sharding: Optimize sharding table index name rewriting rules and remove unnecessary suffix rewriting - [#31171](https://github.com/apache/shardingsphere/issues/31171)

### Bug Fixes

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ void assertStart() throws IOException {
private ContextManager mockContextManager() {
MetaDataContexts metaDataContexts = MetaDataContextsFactory.create(mock(MetaDataPersistService.class), new ShardingSphereMetaData());
ComputeNodeInstanceContext computeNodeInstanceContext = new ComputeNodeInstanceContext(
new ComputeNodeInstance(mock(InstanceMetaData.class)), new StandaloneWorkerIdGenerator(), new ModeConfiguration("Standalone", null),
mock(LockContext.class), new EventBusContext());
new ComputeNodeInstance(mock(InstanceMetaData.class)), new ModeConfiguration("Standalone", null), new EventBusContext());
computeNodeInstanceContext.init(new StandaloneWorkerIdGenerator(), mock(LockContext.class));
return new ContextManager(metaDataContexts, computeNodeInstanceContext, mock(PersistRepository.class));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ public enum MySQLBinaryColumnType implements BinaryColumnType {
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.DATE, DATE);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIME, TIME);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIMESTAMP, TIMESTAMP);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, VAR_STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, VAR_STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, LONG_BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, TINY_BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.NULL, NULL);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BLOB, BLOB);
for (MySQLBinaryColumnType each : values()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ public enum MySQLCharacterSet {
UTF32_GENERAL_CI(60, () -> Charset.forName("utf32")),
UTF32_BIN(61, () -> Charset.forName("utf32")),
UTF16LE_BIN(62, () -> StandardCharsets.UTF_16LE),
BINARY(63, () -> Charset.forName("binary")),
ARMSCII8_BIN(64, () -> Charset.forName("armscii8")),
ASCII_BIN(65, () -> StandardCharsets.US_ASCII),
CP1250_BIN(66, () -> Charset.forName("cp1250")),
Expand Down
Loading

0 comments on commit ce175f9

Please sign in to comment.