diff --git a/src/List.tsx b/src/List.tsx
index 0da61f32..b4717749 100644
--- a/src/List.tsx
+++ b/src/List.tsx
@@ -278,10 +278,12 @@ export function RawList<T>(props: ListProps<T>, ref: React.Ref<ListRef>) {
   React.useLayoutEffect(() => {
     const changedRecord = heights.getRecord();
     if (changedRecord.size === 1) {
-      const recordKey = Array.from(changedRecord)[0];
+      const recordKey = Array.from(changedRecord.keys())[0];
+      const prevCacheHeight = changedRecord.get(recordKey);
+
       // Quick switch data may cause `start` not in `mergedData` anymore
       const startItem = mergedData[start];
-      if (startItem) {
+      if (startItem && prevCacheHeight === undefined) {
         const startIndexKey = getKey(startItem);
         if (startIndexKey === recordKey) {
           const realStartHeight = heights.get(recordKey);
diff --git a/src/utils/CacheMap.ts b/src/utils/CacheMap.ts
index e3d2cef2..abfe019c 100644
--- a/src/utils/CacheMap.ts
+++ b/src/utils/CacheMap.ts
@@ -8,16 +8,18 @@ class CacheMap {
   // `useMemo` no need to update if `id` not change
   id: number = 0;
 
-  diffKeys = new Set<React.Key>();
+  diffRecords = new Map<React.Key, number>();
 
   constructor() {
     this.maps = Object.create(null);
   }
 
   set(key: React.Key, value: number) {
+    // Record prev value
+    this.diffRecords.set(key, this.maps[key as string]);
+
     this.maps[key as string] = value;
     this.id += 1;
-    this.diffKeys.add(key as string);
   }
 
   get(key: React.Key) {
@@ -29,11 +31,11 @@ class CacheMap {
    * To help to know what's update in the next render.
    */
   resetRecord() {
-    this.diffKeys.clear();
+    this.diffRecords.clear();
   }
 
   getRecord() {
-    return this.diffKeys;
+    return this.diffRecords;
   }
 }
 
diff --git a/tests/scroll.test.js b/tests/scroll.test.js
index 1863d41d..e3a7e045 100644
--- a/tests/scroll.test.js
+++ b/tests/scroll.test.js
@@ -38,10 +38,16 @@ describe('List.Scroll', () => {
   beforeAll(() => {
     mockElement = spyElementPrototypes(HTMLElement, {
       offsetHeight: {
-        get: () => 20,
+        get() {
+          const height = this.getAttribute('data-height');
+          return Number(height || 20);
+        },
       },
       clientHeight: {
-        get: () => 100,
+        get() {
+          const height = this.getAttribute('data-height');
+          return Number(height || 100);
+        },
       },
       getBoundingClientRect: () => boundingRect,
       offsetParent: {
@@ -666,4 +672,63 @@ describe('List.Scroll', () => {
       expect(getScrollTop(container)).toEqual(0);
     });
   });
+
+  it('not scroll jump for item height change', async () => {
+    jest.useFakeTimers();
+
+    const onScroll = jest.fn();
+
+    const listRef = React.createRef();
+    const { container } = genList(
+      {
+        itemHeight: 10,
+        height: 100,
+        data: genData(100),
+        ref: listRef,
+        children: ({ id }) => <li data-id={id}>{id}</li>,
+        onScroll,
+      },
+      render,
+    );
+
+    // first render refresh
+    await act(async () => {
+      onLibResize([
+        {
+          target: container.querySelector('.rc-virtual-list-holder-inner'),
+        },
+      ]);
+
+      await Promise.resolve();
+    });
+
+    await act(async () => {
+      jest.advanceTimersByTime(1000);
+      await Promise.resolve();
+    });
+
+    container.querySelector('li[data-id="0"]').setAttribute('data-height', '30');
+
+    // Force change first row height
+    await act(async () => {
+      boundingRect.height = 110;
+
+      onLibResize([
+        {
+          target: container.querySelector('.rc-virtual-list-holder-inner'),
+        },
+      ]);
+
+      await Promise.resolve();
+    });
+
+    await act(async () => {
+      jest.advanceTimersByTime(1000);
+      await Promise.resolve();
+    });
+
+    expect(onScroll).not.toHaveBeenCalled();
+
+    jest.useRealTimers();
+  });
 });