Skip to content

Instantly share code, notes, and snippets.

@LarryRuane

LarryRuane/10ca73c02cbff5.diff

Created June 17, 2026 23:33
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save LarryRuane/a7715ee402b716ea1f951fd0326cd241 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save LarryRuane/a7715ee402b716ea1f951fd0326cd241 to your computer and use it in GitHub Desktop.
Download ZIP
archiving spent coins to leveldb
Raw
10ca73c02cbff5.diff
diff --git a/src/coins.cpp b/src/coins.cpp
index c403e006c8..2eb673910a 100644
--- a/src/coins.cpp
+++ b/src/coins.cpp
@@ -14,16 +14,24 @@ TRACEPOINT_SEMAPHORE(utxocache, add);
TRACEPOINT_SEMAPHORE(utxocache, spent);
TRACEPOINT_SEMAPHORE(utxocache, uncache);
+constexpr bool lmr_debug{false};
+
CoinsViewEmpty& CoinsViewEmpty::Get()
{
static CoinsViewEmpty instance;
return instance;
}
+// Like GetCoin() except doesn't populate the local cache.
std::optional<Coin> CCoinsViewCache::PeekCoin(const COutPoint& outpoint) const
{
if (auto it{cacheCoins.find(outpoint)}; it != cacheCoins.end()) {
- return it->second.coin.IsSpent() ? std::nullopt : std::optional{it->second.coin};
+ if(lmr_debug) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
+ //return it->second.coin.IsSpent() ? std::nullopt : std::optional{it->second.coin};
+ if (it->second.coin.IsSpent()) {
+ return std::nullopt;
+ }
+ return std::optional{it->second.coin};
}
return base->PeekCoin(outpoint);
}
@@ -32,7 +40,8 @@ CCoinsViewCache::CCoinsViewCache(CCoinsView* in_base, bool deterministic) :
CCoinsViewBacked(in_base), m_deterministic(deterministic),
cacheCoins(0, SaltedOutpointHasher(/*deterministic=*/deterministic), CCoinsMap::key_equal{}, &m_cache_coins_memory_resource)
{
- m_sentinel.second.SelfRef(m_sentinel);
+ m_sentinel.spent.second.SelfRef(m_sentinel.spent);
+ m_sentinel.unspent.second.SelfRef(m_sentinel.unspent);
}
size_t CCoinsViewCache::DynamicMemoryUsage() const {
@@ -61,7 +70,9 @@ CCoinsMap::iterator CCoinsViewCache::FetchCoin(const COutPoint &outpoint) const
std::optional<Coin> CCoinsViewCache::GetCoin(const COutPoint& outpoint) const
{
- if (auto it{FetchCoin(outpoint)}; it != cacheCoins.end() && !it->second.coin.IsSpent()) return it->second.coin;
+ auto it{FetchCoin(outpoint)};
+ if(lmr_debug) if (it != cacheCoins.end()) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
+ if (it != cacheCoins.end() && !it->second.coin.IsSpent()) return it->second.coin;
return std::nullopt;
}
@@ -96,9 +107,9 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi
Assume(TrySub(cachedCoinsUsage, it->second.coin.DynamicMemoryUsage()));
}
it->second.coin = std::move(coin);
- CCoinsCacheEntry::SetDirty(*it, m_sentinel);
+ CCoinsCacheEntry::SetDirty(*it, m_sentinel.unspent);
++m_dirty_count;
- if (fresh) CCoinsCacheEntry::SetFresh(*it, m_sentinel);
+ if (fresh) CCoinsCacheEntry::SetFresh(*it, m_sentinel.unspent);
cachedCoinsUsage += it->second.coin.DynamicMemoryUsage();
TRACEPOINT(utxocache, add,
outpoint.hash.data(),
@@ -109,12 +120,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi
}
void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) {
- const auto mem_usage{coin.DynamicMemoryUsage()};
auto [it, inserted] = cacheCoins.try_emplace(std::move(outpoint), std::move(coin));
if (inserted) {
- CCoinsCacheEntry::SetDirty(*it, m_sentinel);
+ CCoinsCacheEntry::SetDirty(*it, m_sentinel.unspent);
++m_dirty_count;
- cachedCoinsUsage += mem_usage;
+ cachedCoinsUsage += it->second.coin.DynamicMemoryUsage();
+ if(lmr_debug) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
}
}
@@ -146,8 +157,10 @@ bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) {
if (it->second.IsFresh()) {
cacheCoins.erase(it);
} else {
- CCoinsCacheEntry::SetDirty(*it, m_sentinel);
+ it->second.SetClean();
+ CCoinsCacheEntry::SetDirty(*it, m_sentinel.spent);
++m_dirty_count;
+ ++m_spent_count;
it->second.coin.Clear();
}
return true;
@@ -160,6 +173,7 @@ const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const {
if (it == cacheCoins.end()) {
return coinEmpty;
} else {
+ if(lmr_debug) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
return it->second.coin;
}
}
@@ -167,11 +181,13 @@ const Coin& CCoinsViewCache::AccessCoin(const COutPoint &outpoint) const {
bool CCoinsViewCache::HaveCoin(const COutPoint& outpoint) const
{
CCoinsMap::const_iterator it = FetchCoin(outpoint);
+ if(lmr_debug) if (it != cacheCoins.end()) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
return (it != cacheCoins.end() && !it->second.coin.IsSpent());
}
bool CCoinsViewCache::HaveCoinInCache(const COutPoint &outpoint) const {
CCoinsMap::const_iterator it = cacheCoins.find(outpoint);
+ if(lmr_debug) if (it != cacheCoins.end()) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
return (it != cacheCoins.end() && !it->second.coin.IsSpent());
}
@@ -189,12 +205,14 @@ void CCoinsViewCache::SetBestBlock(const uint256& in_block_hash)
void CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& in_block_hash)
{
for (auto it{cursor.Begin()}; it != cursor.End(); it = cursor.NextAndMaybeErase(*it)) {
+ if(lmr_debug) assert(!it->second.coin.IsSpent() || (!it->second.IsFresh() && it->second.IsDirty()));
if (!it->second.IsDirty()) { // TODO a cursor can only contain dirty entries
continue;
}
auto [itUs, inserted]{cacheCoins.try_emplace(it->first)};
if (inserted) {
if (it->second.IsFresh() && it->second.coin.IsSpent()) {
+ Assert(false);
cacheCoins.erase(itUs); // TODO fresh coins should have been removed at spend
} else {
// The parent cache does not have an entry, while the child cache does.
@@ -208,16 +226,25 @@ void CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& in
} else {
entry.coin = it->second.coin;
}
- CCoinsCacheEntry::SetDirty(*itUs, m_sentinel);
++m_dirty_count;
+ if (entry.coin.IsSpent()) {
+ CCoinsCacheEntry::SetDirty(*itUs, m_sentinel.spent);
+ ++m_spent_count;
+ } else {
+ CCoinsCacheEntry::SetDirty(*itUs, m_sentinel.unspent);
+ }
cachedCoinsUsage += entry.coin.DynamicMemoryUsage();
// We can mark it FRESH in the parent if it was FRESH in the child
// Otherwise it might have just been flushed from the parent's cache
// and already exist in the grandparent
- if (it->second.IsFresh()) CCoinsCacheEntry::SetFresh(*itUs, m_sentinel);
+ if (it->second.IsFresh()) {
+ Assume(!itUs->second.coin.IsSpent());
+ CCoinsCacheEntry::SetFresh(*itUs, m_sentinel.unspent);
+ }
}
} else {
// Found the entry in the parent cache
+ if(lmr_debug) assert(!itUs->second.coin.IsSpent() || (!itUs->second.IsFresh() && itUs->second.IsDirty()));
if (it->second.IsFresh() && !itUs->second.coin.IsSpent()) {
// The coin was marked FRESH in the child cache, but the coin
// exists in the parent cache. If this ever happens, it means
@@ -229,12 +256,15 @@ void CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& in
if (itUs->second.IsFresh() && it->second.coin.IsSpent()) {
// The grandparent cache does not have an entry, and the coin
// has been spent. We can just delete it from the parent cache.
+ if(lmr_debug) Assert(!itUs->second.coin.IsSpent());
Assume(TrySub(m_dirty_count, itUs->second.IsDirty()));
Assume(TrySub(cachedCoinsUsage, itUs->second.coin.DynamicMemoryUsage()));
cacheCoins.erase(itUs);
} else {
// A normal modification.
Assume(TrySub(cachedCoinsUsage, itUs->second.coin.DynamicMemoryUsage()));
+ Assume(TrySub(m_dirty_count, itUs->second.IsDirty()));
+ Assume(TrySub(m_spent_count, itUs->second.coin.IsSpent()));
if (cursor.WillErase(*it)) {
// Since this entry will be erased,
// we can move the coin into us instead of copying it
@@ -242,30 +272,47 @@ void CCoinsViewCache::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& in
} else {
itUs->second.coin = it->second.coin;
}
- cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage();
- if (!itUs->second.IsDirty()) {
- CCoinsCacheEntry::SetDirty(*itUs, m_sentinel);
- ++m_dirty_count;
+ bool is_fresh{itUs->second.IsFresh()};
+ itUs->second.SetClean();
+ if (itUs->second.coin.IsSpent()) {
+ CCoinsCacheEntry::SetDirty(*itUs, m_sentinel.spent);
+ ++m_spent_count;
+ if (is_fresh) CCoinsCacheEntry::SetFresh(*itUs, m_sentinel.spent);
+ } else {
+ CCoinsCacheEntry::SetDirty(*itUs, m_sentinel.unspent);
+ if (is_fresh) CCoinsCacheEntry::SetFresh(*itUs, m_sentinel.unspent);
}
+ cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage();
+ ++m_dirty_count;
// NOTE: It isn't safe to mark the coin as FRESH in the parent
// cache. If it already existed and was spent in the parent
// cache then marking it FRESH would prevent that spentness
// from being flushed to the grandparent.
}
}
+ if(lmr_debug) assert(!itUs->second.coin.IsSpent() || (!itUs->second.IsFresh() && itUs->second.IsDirty()));
}
SetBestBlock(in_block_hash);
}
+size_t CCoinsViewCache::BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, size_t spent_count, size_t map_size_on_flush)
+{
+ return 0;
+}
+
void CCoinsViewCache::Flush(bool reallocate_cache)
{
+ m_map_size_at_flush = cacheCoins.size();
auto cursor{CoinsViewCacheCursor(m_dirty_count, m_sentinel, cacheCoins, /*will_erase=*/true)};
base->BatchWrite(cursor, m_block_hash);
Assume(m_dirty_count == 0);
+ m_spent_count = 0;
cacheCoins.clear();
+ // It's best if the map doesn't have to resize the buckets array.
if (reallocate_cache) {
ReallocateCache();
}
+ cacheCoins.reserve(m_map_size_at_flush * 102 / 100);
cachedCoinsUsage = 0;
}
@@ -274,24 +321,35 @@ void CCoinsViewCache::Sync()
auto cursor{CoinsViewCacheCursor(m_dirty_count, m_sentinel, cacheCoins, /*will_erase=*/false)};
base->BatchWrite(cursor, m_block_hash);
Assume(m_dirty_count == 0);
- if (m_sentinel.second.Next() != &m_sentinel) {
+ m_spent_count = 0;
+ if (m_sentinel.unspent.second.Next() != &m_sentinel.unspent) {
/* BatchWrite must clear flags of all entries */
throw std::logic_error("Not all unspent flagged entries were cleared");
}
}
+// Unlike Flush() and Sync(), this is a NON-synchronizing action; can be done any time.
+void CCoinsViewCache::FlushSpents() {
+ size_t copied_count{base->BatchWriteSpent(m_sentinel.spent, cacheCoins, m_spent_count, m_map_size_at_flush)};
+ Assume(TrySub(m_spent_count, copied_count));
+ Assume(TrySub(m_dirty_count, copied_count));
+}
+
void CCoinsViewCache::Reset() noexcept
{
cacheCoins.clear();
cachedCoinsUsage = 0;
m_dirty_count = 0;
+ m_spent_count = 0;
SetBestBlock(uint256::ZERO);
}
void CCoinsViewCache::Uncache(const COutPoint& hash)
{
CCoinsMap::iterator it = cacheCoins.find(hash);
+ // i can't imagine this is ever called on a spent coin!
if (it != cacheCoins.end() && !it->second.IsDirty()) {
+ if(lmr_debug) assert(!it->second.coin.IsSpent());
Assume(TrySub(cachedCoinsUsage, it->second.coin.DynamicMemoryUsage()));
TRACEPOINT(utxocache, uncache,
hash.hash.data(),
@@ -333,9 +391,11 @@ void CCoinsViewCache::SanityCheck() const
{
size_t recomputed_usage = 0;
size_t count_dirty = 0;
+ size_t count_spent = 0;
for (const auto& [_, entry] : cacheCoins) {
if (entry.coin.IsSpent()) {
assert(entry.IsDirty() && !entry.IsFresh()); // A spent coin must be dirty and cannot be fresh
+ ++count_spent;
} else {
assert(entry.IsDirty() || !entry.IsFresh()); // An unspent coin must not be fresh if not dirty
}
@@ -348,7 +408,8 @@ void CCoinsViewCache::SanityCheck() const
}
// Iterate over the linked list of flagged entries.
size_t count_linked = 0;
- for (auto it = m_sentinel.second.Next(); it != &m_sentinel; it = it->second.Next()) {
+ for (auto it = m_sentinel.unspent.second.Next(); it != &m_sentinel.unspent; it = it->second.Next()) {
+ assert(!it->second.coin.IsSpent());
// Verify linked list integrity.
assert(it->second.Next()->second.Prev() == it);
assert(it->second.Prev()->second.Next() == it);
@@ -357,7 +418,23 @@ void CCoinsViewCache::SanityCheck() const
// Count the number of entries actually in the list.
++count_linked;
}
- assert(count_dirty == count_linked && count_dirty == m_dirty_count);
+ size_t count_linked_spent = 0;
+ for (auto it = m_sentinel.spent.second.Next(); it != &m_sentinel.spent; it = it->second.Next()) {
+ assert(it->second.coin.IsSpent());
+ // Verify linked list integrity.
+ assert(it->second.Next()->second.Prev() == it);
+ assert(it->second.Prev()->second.Next() == it);
+ // Verify they are actually flagged; also, spent coins cannot be fresh.
+ assert(it->second.IsDirty());
+ assert(!it->second.IsFresh());
+ // Count the number of entries actually in the list.
+ ++count_linked;
+ ++count_linked_spent;
+ }
+ assert(count_dirty == count_linked);
+ assert(count_dirty == m_dirty_count);
+ assert(count_spent == count_linked_spent);
+ assert(count_spent == m_spent_count);
assert(recomputed_usage == cachedCoinsUsage);
}
diff --git a/src/coins.h b/src/coins.h
index ae7f34f465..64044796cd 100644
--- a/src/coins.h
+++ b/src/coins.h
@@ -206,6 +206,28 @@ public:
// Set sentinel to DIRTY so we can call Next on it
m_flags = DIRTY;
}
+
+ // Move the elements in source to the beginning of the destination; arguments are sentinals.
+ static void Merge(CoinsCachePair* source, CoinsCachePair* dest) {
+ if (source->second.m_next != source) {
+ const auto source_first = source->second.m_next;
+ const auto source_last = source->second.m_prev;
+ const auto dest_first = dest->second.m_next;
+
+ // set the destination's first element to be the source's first element
+ dest->second.m_next = source_first;
+ source_first->second.m_prev = dest;
+
+ // set the source's last element to be the destination's original first element
+ source_last->second.m_next = dest_first;
+ dest_first->second.m_prev = source_last;
+
+ // make the source list empty
+ source->second.m_next = source;
+ source->second.m_prev = source;
+ }
+
+ }
};
/**
@@ -244,6 +266,11 @@ private:
uint256 block_hash;
};
+struct Sentinel {
+ CoinsCachePair unspent;
+ CoinsCachePair spent;
+};
+
/**
* Cursor for iterating over the linked list of flagged entries in CCoinsViewCache.
*
@@ -267,13 +294,19 @@ struct CoinsViewCacheCursor
//! Calling CCoinsMap::clear() afterwards is faster because a CoinsCachePair cannot be coerced back into a
//! CCoinsMap::iterator to be erased, and must therefore be looked up again by key in the CCoinsMap before being erased.
CoinsViewCacheCursor(size_t& dirty_count LIFETIMEBOUND,
- CoinsCachePair& sentinel LIFETIMEBOUND,
+ Sentinel& sentinel LIFETIMEBOUND,
CCoinsMap& map LIFETIMEBOUND,
bool will_erase) noexcept
: m_dirty_count(dirty_count), m_sentinel(sentinel), m_map(map), m_will_erase(will_erase) {}
- inline CoinsCachePair* Begin() const noexcept { return m_sentinel.second.Next(); }
- inline CoinsCachePair* End() const noexcept { return &m_sentinel; }
+ inline CoinsCachePair* Begin() const noexcept {
+ // Transfer all the spent items to the beginning of the unspent, making upspent not
+ // the best name, but iterating this list will return all relevalnt (dirty) coins,
+ // and we do want the spent list to end up empty.
+ CCoinsCacheEntry::Merge(&m_sentinel.spent, &m_sentinel.unspent);
+ return m_sentinel.unspent.second.Next();
+ }
+ inline CoinsCachePair* End() const noexcept { return &m_sentinel.unspent; }
//! Return the next entry after current, possibly erasing current
inline CoinsCachePair* NextAndMaybeErase(CoinsCachePair& current) noexcept
@@ -296,9 +329,10 @@ struct CoinsViewCacheCursor
inline bool WillErase(CoinsCachePair& current) const noexcept { return m_will_erase || current.second.coin.IsSpent(); }
size_t GetDirtyCount() const noexcept { return m_dirty_count; }
size_t GetTotalCount() const noexcept { return m_map.size(); }
+ void MapErase(const COutPoint& outpoint) { m_map.erase(outpoint); }
private:
size_t& m_dirty_count;
- CoinsCachePair& m_sentinel;
+ Sentinel& m_sentinel;
CCoinsMap& m_map;
bool m_will_erase;
};
@@ -335,6 +369,21 @@ public:
//! The passed cursor is used to iterate through the coins.
virtual void BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block_hash) = 0;
+ //! Optionally store some of the given spent coins in a separate volatile container,
+ //! up to the entire list (of length spent_count). (Only the spent coins'
+ //! outpoints need to actually be stored.) Can be called repeatedly, and should
+ //! be efficient if it decides to do nothing. Stored spent coins should be
+ //! removed from the list. The return value is the number of coins stored (and
+ //! removed from the list).
+ //!
+ //! A subsequent call to GetCoin, PeekCoin, or HaveCoin with an outpoint in this
+ //! container will return as if the coin doesn't exist (std::nullopt, std::nullopt,
+ //! or false, respectively).
+ //!
+ //! BatchWrite, in addition to what it already does, will permanently record the
+ //! coins from the spent container as spent, and clear this container.
+ virtual size_t BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, size_t spent_count, size_t map_size_at_flush) = 0;
+
//! Get a cursor to iterate over the whole state. Implementations may return nullptr.
virtual std::unique_ptr<CCoinsViewCursor> Cursor() const = 0;
@@ -363,6 +412,7 @@ public:
{
for (auto it{cursor.Begin()}; it != cursor.End(); it = cursor.NextAndMaybeErase(*it)) { }
}
+ size_t BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, size_t spent_count, size_t map_size_at_flush) override { return 0; }
std::unique_ptr<CCoinsViewCursor> Cursor() const override { return {}; }
size_t EstimateSize() const override { return 0; }
};
@@ -384,6 +434,7 @@ public:
uint256 GetBestBlock() const override { return base->GetBestBlock(); }
std::vector<uint256> GetHeadBlocks() const override { return base->GetHeadBlocks(); }
void BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block_hash) override { base->BatchWrite(cursor, block_hash); }
+ size_t BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, size_t spent_count, size_t map_size_at_flush) override { return base->BatchWriteSpent(sentinel, map, spent_count, map_size_at_flush); }
std::unique_ptr<CCoinsViewCursor> Cursor() const override { return base->Cursor(); }
size_t EstimateSize() const override { return base->EstimateSize(); }
};
@@ -402,8 +453,8 @@ protected:
*/
mutable uint256 m_block_hash;
mutable CCoinsMapMemoryResource m_cache_coins_memory_resource{};
- /* The starting sentinel of the flagged entry circular doubly linked list. */
- mutable CoinsCachePair m_sentinel;
+ /* The starting sentinels (spent and unspent) of the flagged entry circular doubly linked lists. */
+ mutable Sentinel m_sentinel;
mutable CCoinsMap cacheCoins;
/* Cached dynamic memory usage for the inner Coin objects. */
@@ -411,6 +462,11 @@ protected:
/* Running count of dirty Coin cache entries. */
mutable size_t m_dirty_count{0};
+ // length of spent list
+ mutable size_t m_spent_count{0};
+
+ mutable size_t m_map_size_at_flush{0};
+
/**
* Discard all modifications made to this cache without flushing to the base view.
* This can be used to efficiently reuse a cache instance across multiple operations.
@@ -435,6 +491,7 @@ public:
uint256 GetBestBlock() const override;
void SetBestBlock(const uint256& block_hash);
void BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block_hash) override;
+ size_t BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, size_t spent_count, size_t map_size_on_flush) override;
std::unique_ptr<CCoinsViewCursor> Cursor() const override {
throw std::logic_error("CCoinsViewCache cursor iteration not supported.");
}
@@ -497,6 +554,13 @@ public:
*/
void Sync();
+ /**
+ * Memory limit may be close; try to flush some spent coins to a lower
+ * cache (see BatchWriteSpents) to free up some space.
+ * This function may do nothing (but in that case, should be efficient).
+ */
+ void FlushSpents();
+
/**
* Removes the UTXO with the given outpoint from the cache, if it is
* not modified.
diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp
index bf91a750c1..b226f4fdb4 100644
--- a/src/node/chainstatemanager_args.cpp
+++ b/src/node/chainstatemanager_args.cpp
@@ -69,6 +69,7 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage
opts.script_execution_cache_bytes = clamped_size_each;
opts.signature_cache_bytes = clamped_size_each;
}
+ opts.coins_view.chain_type = args.GetChainType();
return {};
}
diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp
index 14ccb1c443..78d32f3bdf 100644
--- a/src/test/coins_tests.cpp
+++ b/src/test/coins_tests.cpp
@@ -95,7 +95,7 @@ public:
}
CCoinsMap& map() const { return cacheCoins; }
- CoinsCachePair& sentinel() const { return m_sentinel; }
+ CoinsCachePair& sentinel() const { return m_sentinel.unspent; }
size_t& usage() const { return cachedCoinsUsage; }
size_t& dirty() const { return m_dirty_count; }
};
@@ -654,11 +654,14 @@ static MaybeCoin GetCoinsMapEntry(const CCoinsMap& map, const COutPoint& outp =
static void WriteCoinsViewEntry(CCoinsView& view, const MaybeCoin& cache_coin)
{
- CoinsCachePair sentinel{};
- sentinel.second.SelfRef(sentinel);
+ Sentinel sentinel{};
+ sentinel.unspent.second.SelfRef(sentinel.unspent);
+ sentinel.spent.second.SelfRef(sentinel.spent);
CCoinsMapMemoryResource resource;
CCoinsMap map{0, CCoinsMap::hasher{}, CCoinsMap::key_equal{}, &resource};
- if (cache_coin) InsertCoinsMapEntry(map, sentinel, *cache_coin);
+ if (cache_coin) {
+ InsertCoinsMapEntry(map, sentinel.unspent, *cache_coin);
+ }
size_t dirty_count{cache_coin && cache_coin->IsDirty()};
auto cursor{CoinsViewCacheCursor(dirty_count, sentinel, map, /*will_erase=*/true)};
view.BatchWrite(cursor, {});
diff --git a/src/txdb.cpp b/src/txdb.cpp
index a41dfd1657..bb9d7a2bf3 100644
--- a/src/txdb.cpp
+++ b/src/txdb.cpp
@@ -24,6 +24,7 @@
static constexpr uint8_t DB_COIN{'C'};
static constexpr uint8_t DB_BEST_BLOCK{'B'};
static constexpr uint8_t DB_HEAD_BLOCKS{'H'};
+static constexpr uint8_t DB_SPENT{'S'};
// Keys used in previous version that might still be found in the DB:
static constexpr uint8_t DB_COINS{'c'};
@@ -41,10 +42,11 @@ bool CCoinsViewDB::NeedsUpgrade()
namespace {
+// key_letter is either DB_COIN (the unspent coins) or DB_SPENT
struct CoinEntry {
COutPoint* outpoint;
- uint8_t key{DB_COIN};
- explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)) {}
+ uint8_t key;
+ explicit CoinEntry(uint8_t key, const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(key) {}
SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); }
};
@@ -54,7 +56,21 @@ struct CoinEntry {
CCoinsViewDB::CCoinsViewDB(DBParams db_params, CoinsViewOptions options) :
m_db_params{std::move(db_params)},
m_options{std::move(options)},
- m_db{std::make_unique<CDBWrapper>(m_db_params)} { }
+ m_db{std::make_unique<CDBWrapper>(m_db_params)}
+{
+ //{static bool go;while(!go);}
+ CDBBatch batch(*m_db);
+ std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
+ pcursor->Seek(DB_SPENT);
+ while (pcursor->Valid()) {
+ COutPoint outpoint;
+ CoinEntry entry(0, &outpoint);
+ if (!pcursor->GetKey(entry) || entry.key != DB_SPENT) break;
+ batch.Erase(entry);
+ pcursor->Next();
+ }
+ m_db->WriteBatch(batch);
+}
void CCoinsViewDB::ResizeCache(size_t new_cache_size)
{
@@ -72,7 +88,12 @@ void CCoinsViewDB::ResizeCache(size_t new_cache_size)
std::optional<Coin> CCoinsViewDB::GetCoin(const COutPoint& outpoint) const
{
- if (Coin coin; m_db->Read(CoinEntry(&outpoint), coin)) {
+ //if (m_spent_count && m_db->Exists(CoinEntry(DB_SPENT, &outpoint))) return std::nullopt;
+ if (m_spent_count && m_db->Exists(CoinEntry(DB_SPENT, &outpoint))) {
+ return std::nullopt;
+ }
+
+ if (Coin coin; m_db->Read(CoinEntry(DB_COIN, &outpoint), coin)) {
Assert(!coin.IsSpent()); // The UTXO database should never contain spent coins
return coin;
}
@@ -86,7 +107,15 @@ std::optional<Coin> CCoinsViewDB::PeekCoin(const COutPoint& outpoint) const
bool CCoinsViewDB::HaveCoin(const COutPoint& outpoint) const
{
- return m_db->Exists(CoinEntry(&outpoint));
+ //return !(m_spent_count && m_db->Exists(CoinEntry(DB_SPENT, &outpoint))) && m_db->Exists(CoinEntry(DB_COIN, &outpoint));
+ if (m_spent_count && m_db->Exists(CoinEntry(DB_SPENT, &outpoint))) {
+ //{static bool go{false}; while(!go);}
+ return false;
+ }
+ if (m_db->Exists(CoinEntry(DB_COIN, &outpoint))) {
+ return true;
+ }
+ return false;
}
uint256 CCoinsViewDB::GetBestBlock() const {
@@ -135,11 +164,47 @@ void CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block
batch.Erase(DB_BEST_BLOCK);
batch.Write(DB_HEAD_BLOCKS, Vector(block_hash, old_tip));
+ LogDebug(BCLog::COINDB, "Start txdb BatchWrite, archived spent %i", m_spent_count);
+ // XXX need to break this into multiple batches
+ {
+ size_t count{0};
+ std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
+ pcursor->Seek(DB_SPENT);
+ while (pcursor->Valid()) {
+ COutPoint outpoint;
+ CoinEntry entry(0, &outpoint);
+ if (!pcursor->GetKey(entry) || entry.key != DB_SPENT) break;
+ batch.Erase(entry);
+ entry.key = DB_COIN;
+ batch.Erase(entry);
+ if (batch.ApproximateSize() > m_options.batch_write_bytes) {
+ LogDebug(BCLog::COINDB, "Writing partial spent batch of %.2f MiB\n", batch.ApproximateSize() / double(1_MiB));
+ m_db->WriteBatch(batch);
+ batch.Clear();
+ // LMR this should probably be refactored to avoid the duplication with below
+ if (m_options.simulate_crash_ratio) {
+ static FastRandomContext rng;
+ if (rng.randrange(m_options.simulate_crash_ratio) == 0) {
+ LogError("Simulating a crash. Goodbye.");
+ _Exit(0);
+ }
+ }
+ }
+ pcursor->Next();
+ ++count;
+ }
+ Assume(count == m_spent_count);
+ m_spent_count = 0;
+ }
+
+ LogDebug(BCLog::COINDB, "Start txdb BatchWrite, main loop");
+ size_t spent_count{0};
for (auto it{cursor.Begin()}; it != cursor.End();) {
if (it->second.IsDirty()) {
- CoinEntry entry(&it->first);
+ CoinEntry entry(DB_COIN, &it->first);
if (it->second.coin.IsSpent()) {
batch.Erase(entry);
+ ++spent_count;
} else {
batch.Write(entry, it->second.coin);
}
@@ -148,7 +213,6 @@ void CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block
it = cursor.NextAndMaybeErase(*it);
if (batch.ApproximateSize() > m_options.batch_write_bytes) {
LogDebug(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.ApproximateSize() / double(1_MiB));
-
m_db->WriteBatch(batch);
batch.Clear();
if (m_options.simulate_crash_ratio) {
@@ -167,7 +231,47 @@ void CCoinsViewDB::BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block
LogDebug(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.ApproximateSize() / double(1_MiB));
m_db->WriteBatch(batch);
- LogDebug(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...", (unsigned int)dirty_count, (unsigned int)count);
+ LogDebug(BCLog::COINDB, "Committed %u changed (%u spent) transaction outputs (out of %u) to coin database...", (unsigned int)dirty_count, (unsigned int)spent_count, (unsigned int)count);
+}
+
+size_t CCoinsViewDB::BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, const size_t spent_count, size_t map_size_at_flush)
+{
+ // make sure we're close to running out of cache space, this is about how many
+ // entries (txos) fit in a flush batch with default dbcache size = 1024 MiB
+ // or about 137 bytes per coin, seems about correct
+ // this results in about an 18 MiB batch, which is reasonable.
+ size_t limit{500*1024};
+ // Only optimize (wait until we can build an efficiently-large batch) if functional tests (REGTEST).
+ if (m_options.chain_type != ChainType::REGTEST) {
+ // make sure we can create a large enough batch to be worthwhile
+ if (spent_count < limit) return 0;
+ // make sure the cache is close to full (based on previous flush event)
+
+ if (!map_size_at_flush || map.size() < map_size_at_flush * 98 / 100) return 0;
+ }
+ if (limit > spent_count) limit = spent_count;
+
+ LogDebug(BCLog::COINDB, "BatchWriteSpent begin map-spent-count: %i", spent_count);
+ CDBBatch batch(*m_db);
+ size_t count{0};
+ while (count < limit && sentinel.second.Next() != &sentinel) {
+ CoinsCachePair& pair{*sentinel.second.Next()};
+ //LogDebug(BCLog::COINDB, "BatchWriteSpent OP %s", pair.first.ToString());
+ assert(pair.second.coin.IsSpent());
+ Assert(pair.second.IsDirty());
+ Assert(!pair.second.IsFresh());
+ CoinEntry entry(DB_SPENT, &pair.first);
+ // no associated data
+ batch.Write(entry, std::vector<uint8_t>());
+ size_t n = map.erase(pair.first);
+ assert(n);
+ if (!n) break;
+ ++count;
+ }
+ m_spent_count += count;
+ m_db->WriteBatch(batch);
+ LogDebug(BCLog::COINDB, "BatchWriteSpent wrote:%i on-disk-spent-count: %i batch: %.2f MiB", count, m_spent_count, batch.ApproximateSize()/(double(1_MiB)));
+ return count;
}
size_t CCoinsViewDB::EstimateSize() const
@@ -208,7 +312,7 @@ std::unique_ptr<CCoinsViewCursor> CCoinsViewDB::Cursor() const
i->pcursor->Seek(DB_COIN);
// Cache key of first record
if (i->pcursor->Valid()) {
- CoinEntry entry(&i->keyTmp.second);
+ CoinEntry entry(DB_COIN, &i->keyTmp.second);
i->pcursor->GetKey(entry);
i->keyTmp.first = entry.key;
} else {
@@ -240,7 +344,7 @@ bool CCoinsViewDBCursor::Valid() const
void CCoinsViewDBCursor::Next()
{
pcursor->Next();
- CoinEntry entry(&keyTmp.second);
+ CoinEntry entry(DB_COIN, &keyTmp.second);
if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
} else {
diff --git a/src/txdb.h b/src/txdb.h
index b19b312a4b..0abbfa4120 100644
--- a/src/txdb.h
+++ b/src/txdb.h
@@ -11,6 +11,7 @@
#include <kernel/caches.h>
#include <kernel/cs_main.h>
#include <sync.h>
+#include <util/chaintype.h>
#include <util/fs.h>
#include <cstddef>
@@ -28,6 +29,7 @@ struct CoinsViewOptions {
size_t batch_write_bytes{DEFAULT_DB_CACHE_BATCH};
//! If non-zero, randomly exit when the database is flushed with (1/ratio) probability.
int simulate_crash_ratio{0};
+ ChainType chain_type{ChainType::MAIN};
};
/** CCoinsView backed by the coin database (chainstate/) */
@@ -37,6 +39,8 @@ protected:
DBParams m_db_params;
CoinsViewOptions m_options;
std::unique_ptr<CDBWrapper> m_db;
+ // m_spent_count is a performance opt that allows a coin lookup to skip the spent table ('S') if 0.
+ size_t m_spent_count{0};
public:
explicit CCoinsViewDB(DBParams db_params, CoinsViewOptions options);
@@ -46,6 +50,7 @@ public:
uint256 GetBestBlock() const override;
std::vector<uint256> GetHeadBlocks() const override;
void BatchWrite(CoinsViewCacheCursor& cursor, const uint256& block_hash) override;
+ size_t BatchWriteSpent(CoinsCachePair& sentinel, CCoinsMap& map, size_t spent_count, size_t map_size_at_flush) override;
std::unique_ptr<CCoinsViewCursor> Cursor() const override;
//! Whether an unsupported database format is used.
diff --git a/src/validation.cpp b/src/validation.cpp
index 211a8122a9..ec5f95ee6b 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -7,6 +7,8 @@
#include <validation.h>
+#include <fstream>
+
#include <arith_uint256.h>
#include <chain.h>
#include <checkqueue.h>
@@ -2756,7 +2758,11 @@ bool Chainstate::FlushStateToDisk(
}
}
const auto nNow{NodeClock::now()};
- // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing).
+ // this returns immediately if there's nothing to do
+ if (mode == FlushStateMode::PERIODIC) {
+ CoinsTip().FlushSpents();
+ }
+ // The cache is large and we're within 20% and 10 MiB of the limit, but we have time now (not in the middle of a block processing).
bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE;
// The cache is over the limit, we have to write now.
bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL;
@@ -2767,8 +2773,8 @@ bool Chainstate::FlushStateToDisk(
bool should_write = (mode == FlushStateMode::FORCE_SYNC) || empty_cache || fPeriodicWrite || fFlushForPrune;
// Write blocks, block index and best chain related state to disk.
if (should_write) {
- LogDebug(BCLog::COINDB, "Writing chainstate to disk: flush mode=%s, prune=%d, large=%d, critical=%d, periodic=%d",
- FlushStateModeNames[size_t(mode)], fFlushForPrune, fCacheLarge, fCacheCritical, fPeriodicWrite);
+ LogDebug(BCLog::COINDB, "Writing chainstate to disk: flush mode=%s, prune=%d, large=%d, critical=%d, periodic=%d, empty=%d",
+ FlushStateModeNames[size_t(mode)], fFlushForPrune, fCacheLarge, fCacheCritical, fPeriodicWrite, empty_cache);
// Ensure we can write block index
if (!CheckDiskSpace(m_blockman.m_opts.blocks_dir)) {
@@ -2816,6 +2822,18 @@ bool Chainstate::FlushStateToDisk(
(uint64_t)coins_count,
(uint64_t)coins_mem_usage,
(bool)fFlushForPrune);
+
+ /*
+ fs::path statm_path{"/proc/self/statm"};
+ std::ifstream statm_file(statm_path.std_path());
+ if (statm_file.is_open()) {
+ uint64_t size;
+ uint64_t resident;
+ if (statm_file >> size >> resident) {
+ LogDebug(BCLog::COINDB, "/proc RSS %u KiB %u MiB", resident * 4096 / 1024, resident * 4096 / 1024 / 1024);
+ }
+ }
+ */
}
}
@@ -2864,7 +2882,12 @@ static void UpdateTipLog(
AssertLockHeld(::cs_main);
// Disable rate limiting in LogPrintLevel_ so this source location may log during IBD.
- LogPrintLevel_(BCLog::LogFlags::ALL, util::log::Level::Info, /*should_ratelimit=*/false, "%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n",
+ fs::path statm_path{"/proc/self/statm"};
+ std::ifstream statm_file(statm_path.std_path());
+ uint64_t size{0};
+ uint64_t resident{0};
+ if (statm_file.is_open()) statm_file >> size >> resident;
+ LogPrintLevel_(BCLog::LogFlags::ALL, util::log::Level::Info, /*should_ratelimit=*/false, "%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s pages=%i\n",
prefix, func_name,
tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion,
log(tip->nChainWork.getdouble()) / log(2.0), tip->m_chain_tx_count,
@@ -2872,7 +2895,8 @@ static void UpdateTipLog(
background_validation ? chainman.GetBackgroundVerificationProgress(*tip) : chainman.GuessVerificationProgress(tip),
coins_tip.DynamicMemoryUsage() / double(1_MiB),
coins_tip.GetCacheSize(),
- !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : "");
+ !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : "",
+ resident);
}
void Chainstate::UpdateTip(const CBlockIndex* pindexNew)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment