[TODO] Remove ChainState: (3) Improve tracking of Consensus tip with ChainIndexer tip

src/NBitcoin.Tests/ChainTests.cs

@@ -207,7 +207,7 @@ public void CanEnumerateAfterChainedBlock()
             enumerator.MoveNext();
             Assert.True(enumerator.Current == c);
 
-            chain.Initialize(b);
+            chain.SetTip(b);
             ChainedHeader cc = this.AppendBlock(chain);
             ChainedHeader dd = this.AppendBlock(chain);
 
@@ -375,19 +375,5 @@ private ChainedHeader AppendBlock(params ChainIndexer[] chainsIndexer)
             ChainedHeader index = null;
             return this.AppendBlock(index, chainsIndexer);
         }
-
-        /// <summary>
-        /// Returns the first common chained block header between two chains.
-        /// </summary>
-        /// <param name="chainSrc">The source chain.</param>
-        /// <param name="otherChain">The other chain.</param>
-        /// <returns>First common chained block header or <c>null</c>.</returns>
-        private ChainedHeader FindFork(ChainIndexer chainSrc, ChainIndexer otherChain)
-        {
-            if (otherChain == null)
-                throw new ArgumentNullException("otherChain");
-
-            return chainSrc.FindFork(otherChain.Tip.EnumerateToGenesis().Select(o => o.HashBlock));
-        }
     }
 }

src/NBitcoin/ChainIndexer.cs

@@ -1,5 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("Stratis.Bitcoin.Tests.Common")]
+[assembly: InternalsVisibleTo("Stratis.SmartContracts.Core.Tests")]
+[assembly: InternalsVisibleTo("Stratis.Bitcoin.Features.PoA.Tests")]
 
 namespace NBitcoin
 {
@@ -28,6 +34,7 @@ public class ChainIndexer
         /// The tip height of the best known validated chain.
         /// </summary>
         public int Height => this.Tip.Height;
+
         public ChainedHeader Genesis => this.GetHeader(0);
 
         public ChainIndexer()
@@ -37,46 +44,36 @@ public ChainIndexer()
         }
 
         public ChainIndexer(Network network) : this()
-        {   
-            this.Network = network;
-
-            this.Initialize(new ChainedHeader(network.GetGenesis().Header, network.GetGenesis().GetHash(), 0));
-        }
-
-        public ChainIndexer(Network network, ChainedHeader chainedHeader) : this()
         {
             this.Network = network;
 
-            this.Initialize(chainedHeader);
+            var tip = new ChainedHeader(this.Network.GetGenesis().Header, this.Network.GetGenesis().GetHash(), 0);
+            AddInternal(tip);
+
+            this.Tip = tip;
         }
 
-        public void Initialize(ChainedHeader chainedHeader)
+        public ChainedHeader SetTip(ChainedHeader chainedHeader)
         {
             lock (this.lockObject)
             {
-                this.blocksById.Clear();
-                this.blocksByHeight.Clear();
+                ChainedHeader fork = chainedHeader.FindFork(this.Tip);
 
-                ChainedHeader iterator = chainedHeader;
+                if (fork == null)
+                    throw new InvalidOperationException("Wrong network");
 
-                while (iterator != null)
-                {
-                    this.blocksById.Add(iterator.HashBlock, iterator);
-                    this.blocksByHeight.Add(iterator.Height, iterator);
+                foreach (ChainedHeader header in this.Tip.EnumerateToGenesis().TakeWhile(h => h.Height > fork.Height))
+                    RemoveInternal(header);
 
-                    if (iterator.Height == 0)
-                    {
-                        if (this.Network.GenesisHash != iterator.HashBlock)
-                            throw new InvalidOperationException("Wrong network");
-                    }
-
-                    iterator = iterator.Previous;
-                }
+                foreach (ChainedHeader header in chainedHeader.EnumerateToGenesis().TakeWhile(h => h.Height > fork.Height))
+                    AddInternal(header);
 
                 this.Tip = chainedHeader;
+
+                return fork;
             }
         }
-        
+
         /// <summary>
         /// Returns the first chained block header that exists in the chain from the list of block hashes.
         /// </summary>
@@ -87,15 +84,18 @@ public ChainedHeader FindFork(IEnumerable<uint256> hashes)
             if (hashes == null)
                 throw new ArgumentNullException("hashes");
 
-            // Find the first block the caller has in the main chain.
-            foreach (uint256 hash in hashes)
+            lock (this.lockObject)
             {
-                ChainedHeader chainedHeader = this.GetHeader(hash);
-                if (chainedHeader != null)
-                    return chainedHeader;
-            }
+                // Find the first block the caller has in the main chain.
+                foreach (uint256 hash in hashes)
+                {
+                    ChainedHeader chainedHeader = this.GetHeader(hash);
+                    if (chainedHeader != null)
+                        return chainedHeader;
+                }
 
-            return null;
+                return null;
+            }
         }
 
         /// <summary>
@@ -111,26 +111,12 @@ public ChainedHeader FindFork(BlockLocator locator)
             return this.FindFork(locator.Blocks);
         }
 
-        /// <summary>
-        /// Enumerate chain block headers after given block hash to genesis block.
-        /// </summary>
-        /// <param name="blockHash">Block hash to enumerate after.</param>
-        /// <returns>Enumeration of chained block headers after given block hash.</returns>
-        public IEnumerable<ChainedHeader> EnumerateAfter(uint256 blockHash)
-        {
-            ChainedHeader block = this.GetHeader(blockHash);
-
-            if (block == null)
-                return new ChainedHeader[0];
-
-            return this.EnumerateAfter(block);
-        }
-
         /// <summary>
         /// Enumerates chain block headers from the given chained block header to tip.
         /// </summary>
         /// <param name="block">Chained block header to enumerate from.</param>
         /// <returns>Enumeration of chained block headers from given chained block header to tip.</returns>
+        /// <remarks>The chain could re-org in which case the enumeration may exit early when encountering a block from a different chain.</remarks>
         public IEnumerable<ChainedHeader> EnumerateToTip(ChainedHeader block)
         {
             if (block == null)
@@ -144,74 +130,78 @@ public IEnumerable<ChainedHeader> EnumerateToTip(ChainedHeader block)
         /// </summary>
         /// <param name="blockHash">Block hash to enumerate from.</param>
         /// <returns>Enumeration of chained block headers from the given block hash to tip.</returns>
+        /// <remarks>The chain could re-org in which case the enumeration may exit early when encountering a block from a different chain.</remarks>
         public IEnumerable<ChainedHeader> EnumerateToTip(uint256 blockHash)
         {
-            ChainedHeader block = this.GetHeader(blockHash);
+            ChainedHeader block = this[blockHash];
             if (block == null)
-                yield break;
-
-            yield return block;
+                return new ChainedHeader[0];
 
-            foreach (ChainedHeader chainedBlock in this.EnumerateAfter(blockHash))
-                yield return chainedBlock;
+            return EnumerateAfter(block).Prepend(block);
         }
 
         /// <summary>
         /// Enumerates chain block headers after the given chained block header to genesis block.
         /// </summary>
         /// <param name="block">The chained block header to enumerate after.</param>
         /// <returns>Enumeration of chained block headers after the given block.</returns>
-        public virtual IEnumerable<ChainedHeader> EnumerateAfter(ChainedHeader block)
+        /// <remarks>The chain could re-org in which case the enumeration may exit early when encountering a block from a different chain.</remarks>
+        internal virtual IEnumerable<ChainedHeader> EnumerateAfter(ChainedHeader block)
         {
-            int i = block.Height + 1;
-            ChainedHeader prev = block;
-
-            while (true)
+            for (int i = block.Height + 1; i <= this.Tip.Height; i++)
             {
-                ChainedHeader b = this.GetHeader(i);
-                if ((b == null) || (b.Previous != prev))
-                    yield break;
+                lock (this.lockObject)
+                {
+                    ChainedHeader nextBlock = this.blocksByHeight[i];
+
+                    if (nextBlock.Previous != block)
+                        yield break;
+
+                    block = nextBlock;
+                }
 
-                yield return b;
-                i++;
-                prev = b;
+                yield return block;
             }
         }
 
-        /// <summary>
-        /// TODO: Make this internal when the component moves to Stratis.Bitcoin
-        /// </summary>
-        public void Add(ChainedHeader addTip)
+        internal void Add(ChainedHeader addTip)
         {
             lock (this.lockObject)
             {
-                if(this.Tip.HashBlock != addTip.Previous.HashBlock)
+                if (this.Tip.HashBlock != addTip.Previous.HashBlock)
                     throw new InvalidOperationException("New tip must be consecutive");
 
-                this.blocksById.Add(addTip.HashBlock, addTip);
-                this.blocksByHeight.Add(addTip.Height, addTip);
+                AddInternal(addTip);
 
                 this.Tip = addTip;
             }
         }
 
-        /// <summary>
-        /// TODO: Make this internal when the component moves to Stratis.Bitcoin
-        /// </summary>
-        public void Remove(ChainedHeader removeTip)
+        private void AddInternal(ChainedHeader addTip)
+        {
+            this.blocksById.Add(addTip.HashBlock, addTip);
+            this.blocksByHeight.Add(addTip.Height, addTip);
+        }
+
+        internal void Remove(ChainedHeader removeTip)
         {
             lock (this.lockObject)
             {
                 if (this.Tip.HashBlock != removeTip.HashBlock)
                     throw new InvalidOperationException("Trying to remove item that is not the tip.");
 
-                this.blocksById.Remove(removeTip.HashBlock);
-                this.blocksByHeight.Remove(removeTip.Height);
+                RemoveInternal(removeTip);
 
                 this.Tip = this.blocksById[removeTip.Previous.HashBlock];
             }
         }
 
+        private void RemoveInternal(ChainedHeader removeTip)
+        {
+            this.blocksById.Remove(removeTip.HashBlock);
+            this.blocksByHeight.Remove(removeTip.Height);
+        }
+
         /// <summary>
         /// Get a <see cref="ChainedHeader"/> based on it's hash.
         /// </summary>

src/Stratis.Bitcoin.Features.BlockStore/BlockStoreQueue.cs

@@ -427,7 +427,7 @@ private ChainedHeader RecoverStoreTip()
             ChainedHeader newTip = this.chainIndexer[firstNotFound - 1];
 
             // Set chain store to be same as the store tip.
-            this.chainIndexer.Initialize(newTip);
+            this.chainIndexer.SetTip(newTip);
 
             this.logger.LogWarning("Block store tip recovered to block '{0}'.", newTip);
 

src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenHeaderConsenusManagerBehaviorTests.cs

@@ -87,7 +87,8 @@ public void ConstructProvenHeaderPayload_Consecutive_Headers()
         {
             var provenHeaderChain = BuildProvenHeaderChain(10);
 
-            var chain = new ChainIndexer(this.Network, provenHeaderChain);
+            var chain = new ChainIndexer(this.Network);
+            chain.SetTip(provenHeaderChain);
 
             var consensusManager = new Mock<IConsensusManager>();
             consensusManager.Setup(c => c.Tip).Returns(provenHeaderChain);

src/Stratis.Bitcoin.Tests.Common/ConsensusChainIndexerExtensions.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
 using NBitcoin;
 using Xunit;
@@ -35,48 +34,6 @@ public static bool TrySetTip(this ChainIndexer chainIndexer, BlockHeader header,
             return true;
         }
 
-        public static ChainedHeader SetTip(this ChainIndexer chainIndexer, ChainedHeader block)
-        {
-            ChainedHeader fork = chainIndexer.Tip.FindFork(block);
-
-            chainIndexer.Initialize(block);
-
-            return fork;
-        }
-
-        private static IEnumerable<ChainedHeader> EnumerateThisToFork(this ChainIndexer chainIndexer, ChainedHeader block)
-        {
-            if (chainIndexer.Tip == null)
-                yield break;
-
-            ChainedHeader tip = chainIndexer.Tip;
-            while (true)
-            {
-                if (ReferenceEquals(null, block) || ReferenceEquals(null, tip))
-                    throw new InvalidOperationException("No fork found between the two chains");
-
-                if (tip.Height > block.Height)
-                {
-                    yield return tip;
-                    tip = tip.Previous;
-                }
-                else if (tip.Height < block.Height)
-                {
-                    block = block.Previous;
-                }
-                else if (tip.Height == block.Height)
-                {
-                    if (tip.HashBlock == block.HashBlock)
-                        break;
-
-                    yield return tip;
-
-                    block = block.Previous;
-                    tip = tip.Previous;
-                }
-            }
-        }
-
         public static ChainIndexer Load(this ChainIndexer chainIndexer, byte[] chain)
         {
             using (var ms = new MemoryStream(chain))

src/Stratis.Bitcoin.Tests/Consensus/ConsensusTestContextBuilder.cs

@@ -33,7 +33,7 @@ internal TestContext Build()
             {
                 this.testContext.coinView.UpdateTipHash(new HashHeightPair(this.testContext.InitialChainTip));
                 this.testContext.ChainedHeaderTree.Initialize(this.testContext.InitialChainTip);
-                this.testContext.chainIndexer.Initialize(this.testContext.InitialChainTip);
+                this.testContext.chainIndexer.SetTip(this.testContext.InitialChainTip);
                 this.testContext.ChainState.Setup(c => c.BlockStoreTip)
                     .Returns(this.testContext.InitialChainTip);
             }

src/Stratis.Bitcoin/Base/BaseFeature.cs

@@ -259,7 +259,7 @@ public override async Task InitializeAsync()
             }
 
             if (this.chainIndexer.Tip.Height != initializedAt.Height)
-                this.chainIndexer.Initialize(initializedAt);
+                this.chainIndexer.SetTip(initializedAt);
 
             NetworkPeerConnectionParameters connectionParameters = this.connectionManager.Parameters;
             connectionParameters.IsRelay = this.connectionManager.ConnectionSettings.RelayTxes;
@@ -330,7 +330,7 @@ private async Task StartChainAsync()
 
             this.logger.LogInformation("Loading chain.");
             ChainedHeader chainTip = await this.chainRepository.LoadAsync(this.chainIndexer.Genesis).ConfigureAwait(false);
-            this.chainIndexer.Initialize(chainTip);
+            this.chainIndexer.SetTip(chainTip);
 
             this.logger.LogInformation("Chain loaded at height {0}.", this.chainIndexer.Height);