From b3387cc1a11503b2d2e8f4aaf8952c0096ba4c40 Mon Sep 17 00:00:00 2001 From: shi yuhang <52435083+shiyuhang0@users.noreply.github.com> Date: Tue, 1 Aug 2023 15:28:07 +0800 Subject: [PATCH 1/4] This is an automated cherry-pick of #2740 Signed-off-by: ti-chi-bot --- .../org/apache/spark/sql/IssueTestSuite.scala | 41 +++++++++++++++++++ .../tikv/columnar/TiBlockColumnVector.java | 9 ++++ 2 files changed, 50 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala index b3b8da02ad..16183ca3e9 100644 --- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala @@ -22,6 +22,47 @@ import org.apache.spark.sql.functions.{col, sum} class IssueTestSuite extends BaseTiSparkTest { +<<<<<<< HEAD +======= + test("test tiflash overflow in unsigned bigint") { + if (!enableTiFlashTest) { + cancel("tiflash test not enabled") + } + val dbTable = "tispark_test.tiflash_overflow" + tidbStmt.execute(s"drop table if exists $dbTable") + tidbStmt.execute( + s"CREATE TABLE $dbTable (`a` bigint(20) UNSIGNED NOT NULL,`b` bigint(20) UNSIGNED NOT NULL)") + tidbStmt.execute(s"insert into $dbTable values(16717361816800086255, 16717361816800086255)") + tidbStmt.execute(s"ALTER TABLE $dbTable SET TIFLASH REPLICA 1") + + Thread.sleep(5 * 1000) + + val prev = spark.conf.getOption(TiConfigConst.ISOLATION_READ_ENGINES) + try { + spark.conf + .set(TiConfigConst.ISOLATION_READ_ENGINES, TiConfigConst.TIFLASH_STORAGE_ENGINE) + val df = spark.sql(s"select * from $dbTable") + val row = Row(BigDecimal("16717361816800086255"), BigDecimal("16717361816800086255")) + checkAnswer(df, Seq(row)) + } finally { + spark.conf.set( + TiConfigConst.ISOLATION_READ_ENGINES, + prev.getOrElse(TiConfigConst.DEFAULT_STORAGE_ENGINES)) + } + } + + test("test issue 2649") { + val dbTable = "tispark_test.mutil_uniq" + tidbStmt.execute(s"drop table if exists $dbTable") + tidbStmt.execute( + s"CREATE TABLE $dbTable (`a` int(5) NOT NULL,`b` int(5) NOT NULL,UNIQUE KEY `idx_ab` (`a`,`b`))") + tidbStmt.execute(s"insert into $dbTable values(0, 0),(1,1)") + val df = spark.sql(s"select * from $dbTable where a=1") + val row1 = Row(1, 1) + checkAnswer(df, Seq(row1)) + } + +>>>>>>> 044630939 (Fix TiFlash overflow (#2740)) //https://github.com/pingcap/tispark/issues/2268 test("show rowid in commonhandle") { spark.sqlContext.setConf(TiConfigConst.SHOW_ROWID, "true") diff --git a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java index c683776dcc..9436d7e561 100644 --- a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java +++ b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java @@ -18,13 +18,18 @@ import static com.pingcap.tikv.util.MemoryUtil.EMPTY_BYTE_BUFFER_DIRECT; +<<<<<<< HEAD import com.pingcap.tikv.ExtendedDateTime; +======= +import com.google.common.primitives.UnsignedLong; +>>>>>>> 044630939 (Fix TiFlash overflow (#2740)) import com.pingcap.tikv.codec.Codec.DateCodec; import com.pingcap.tikv.codec.Codec.DateTimeCodec; import com.pingcap.tikv.columnar.datatypes.CHType; import com.pingcap.tikv.types.AbstractDateTimeType; import com.pingcap.tikv.types.BytesType; import com.pingcap.tikv.types.DateType; +import com.pingcap.tikv.types.DecimalType; import com.pingcap.tikv.util.MemoryUtil; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -237,6 +242,10 @@ public double getDouble(int rowId) { @Override public BigDecimal getDecimal(int rowId, int precision, int scale) { long rowIdAddr = (long) rowId * fixedLength + dataAddr; + // avoid unsigned long overflow here + if (type == DecimalType.BIG_INT_DECIMAL) { + return new BigDecimal(UnsignedLong.fromLongBits(this.getLong(rowId)).bigIntegerValue()); + } if (fixedLength == 4) { return MemoryUtil.getDecimal32(rowIdAddr, scale); } else if (fixedLength == 8) { From 5be38951d277a14703adc2543fd6b7406c3639e6 Mon Sep 17 00:00:00 2001 From: shi yuhang <52435083+shiyuhang0@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:13:19 +0800 Subject: [PATCH 2/4] Update IssueTestSuite.scala --- core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala index 16183ca3e9..e510bb17a3 100644 --- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala @@ -22,8 +22,6 @@ import org.apache.spark.sql.functions.{col, sum} class IssueTestSuite extends BaseTiSparkTest { -<<<<<<< HEAD -======= test("test tiflash overflow in unsigned bigint") { if (!enableTiFlashTest) { cancel("tiflash test not enabled") @@ -62,7 +60,6 @@ class IssueTestSuite extends BaseTiSparkTest { checkAnswer(df, Seq(row1)) } ->>>>>>> 044630939 (Fix TiFlash overflow (#2740)) //https://github.com/pingcap/tispark/issues/2268 test("show rowid in commonhandle") { spark.sqlContext.setConf(TiConfigConst.SHOW_ROWID, "true") From 2e3d8cca8bfd1e1ca97e99840cf99526434faad0 Mon Sep 17 00:00:00 2001 From: shi yuhang <52435083+shiyuhang0@users.noreply.github.com> Date: Wed, 2 Aug 2023 13:13:42 +0800 Subject: [PATCH 3/4] Update TiBlockColumnVector.java --- .../java/com/pingcap/tikv/columnar/TiBlockColumnVector.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java index 9436d7e561..f2f6e3c0e6 100644 --- a/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java +++ b/tikv-client/src/main/java/com/pingcap/tikv/columnar/TiBlockColumnVector.java @@ -18,11 +18,8 @@ import static com.pingcap.tikv.util.MemoryUtil.EMPTY_BYTE_BUFFER_DIRECT; -<<<<<<< HEAD -import com.pingcap.tikv.ExtendedDateTime; -======= import com.google.common.primitives.UnsignedLong; ->>>>>>> 044630939 (Fix TiFlash overflow (#2740)) +import com.pingcap.tikv.ExtendedDateTime; import com.pingcap.tikv.codec.Codec.DateCodec; import com.pingcap.tikv.codec.Codec.DateTimeCodec; import com.pingcap.tikv.columnar.datatypes.CHType; From 92a082366db4973b9e87ad0c8af23db812ee9f4b Mon Sep 17 00:00:00 2001 From: shi yuhang <52435083+shiyuhang0@users.noreply.github.com> Date: Thu, 3 Aug 2023 10:33:25 +0800 Subject: [PATCH 4/4] Update IssueTestSuite.scala --- .../scala/org/apache/spark/sql/IssueTestSuite.scala | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala index e510bb17a3..3821481572 100644 --- a/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala +++ b/core/src/test/scala/org/apache/spark/sql/IssueTestSuite.scala @@ -49,17 +49,6 @@ class IssueTestSuite extends BaseTiSparkTest { } } - test("test issue 2649") { - val dbTable = "tispark_test.mutil_uniq" - tidbStmt.execute(s"drop table if exists $dbTable") - tidbStmt.execute( - s"CREATE TABLE $dbTable (`a` int(5) NOT NULL,`b` int(5) NOT NULL,UNIQUE KEY `idx_ab` (`a`,`b`))") - tidbStmt.execute(s"insert into $dbTable values(0, 0),(1,1)") - val df = spark.sql(s"select * from $dbTable where a=1") - val row1 = Row(1, 1) - checkAnswer(df, Seq(row1)) - } - //https://github.com/pingcap/tispark/issues/2268 test("show rowid in commonhandle") { spark.sqlContext.setConf(TiConfigConst.SHOW_ROWID, "true")