From 5c5e21be1d70a27f8222826ecc2599a60659ab08 Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 8 Apr 2016 17:08:18 +0100 Subject: [PATCH 1/7] Stop caching, test Mysql Views --- src/Chain/CachingUtxoSet.php | 50 +++++++++++++----------- src/Db.php | 76 ++++++++++++++++++++++++++++++++++++ src/DbInterface.php | 5 ++- src/DebugDb.php | 16 ++++++++ src/Index/Blocks.php | 3 +- 5 files changed, 126 insertions(+), 24 deletions(-) diff --git a/src/Chain/CachingUtxoSet.php b/src/Chain/CachingUtxoSet.php index eac9f22..2444e57 100644 --- a/src/Chain/CachingUtxoSet.php +++ b/src/Chain/CachingUtxoSet.php @@ -30,7 +30,7 @@ class CachingUtxoSet * @var OutPointSerializer */ private $outpointSerializer; - + private $useCaching = false; /** * UtxoSet constructor. * @param DbInterface $db @@ -61,21 +61,24 @@ public function applyBlock(array $deleteOutPoints, array $newUtxos) //if (!empty($this->cacheHits)) { // $this->db->appendUtxoViewKeys($this->cacheHits); // } - - $this->db->updateUtxoSet($this->outpointSerializer, $deleteOutPoints, $newUtxos, $this->cacheHits); + + $this->db->updateUtxoView($this->outpointSerializer, $deleteOutPoints, $newUtxos, []); + //$this->db->updateUtxoSet($this->outpointSerializer, $deleteOutPoints, $newUtxos, $this->cacheHits); }); - foreach ($this->cacheHits as $key) { - $this->set->delete($key); - } + if ($this->useCaching) { + foreach ($this->cacheHits as $key) { + $this->set->delete($key); + } - foreach ($newUtxos as $c => $utxo) { - $new = $this->outpointSerializer->serialize($utxo->getOutPoint())->getBinary(); - $this->set->save($new, [ - $newUtxos[$c]->getOutput()-> getValue(), - $newUtxos[$c]->getOutput()->getScript()->getBinary(), - ], 500000); + foreach ($newUtxos as $c => $utxo) { + $new = $this->outpointSerializer->serialize($utxo->getOutPoint())->getBinary(); + $this->set->save($new, [ + $newUtxos[$c]->getOutput()-> getValue(), + $newUtxos[$c]->getOutput()->getScript()->getBinary(), + ], 500000); + } } echo "Inserts: " . count($newUtxos). " | Deletes: " . count($deleteOutPoints). " | " . "CacheHits: " . count($this->cacheHits) .PHP_EOL; @@ -96,26 +99,29 @@ public function fetchView(array $requiredOutpoints) $a = 0; $b = 0; foreach ($requiredOutpoints as $c => $outpoint) { - $key = $this->outpointSerializer->serialize($outpoint)->getBinary(); - if ($this->set->contains($key)) { - list ($value, $scriptPubKey) = $this->set->fetch($key); - $cacheHits[] = $key; - $utxos[] = new Utxo($outpoint, new TransactionOutput($value, new Script(new Buffer($scriptPubKey)))); - $a++; - } else { - $required[] = $outpoint; - $b++; + if ($this->useCaching) { + $key = $this->outpointSerializer->serialize($outpoint)->getBinary(); + if ($this->set->contains($key)) { + list ($value, $scriptPubKey) = $this->set->fetch($key); + $cacheHits[] = $key; + $utxos[] = new Utxo($outpoint, new TransactionOutput($value, new Script(new Buffer($scriptPubKey)))); + $a++; + continue; + } } + + $required[] = $outpoint; } if (empty($required) === false) { - $utxos = array_merge($utxos, $this->db->fetchUtxoDbList($this->outpointSerializer, $required)); + $utxos = array_merge($utxos, $this->db->createMiniUtxoView($this->outpointSerializer, $required)); } $this->cacheHits = $cacheHits; return $utxos; } catch (\Exception $e) { + echo $e->getMessage() . PHP_EOL . $e->getTraceAsString().PHP_EOL; throw new \RuntimeException('Failed to find UTXOS in set'); } } diff --git a/src/Db.php b/src/Db.php index fc2ff2e..ed9eed1 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1062,6 +1062,82 @@ private function createInsertJoinSql(OutPointSerializer $serializer, array $outp return "INSERT INTO outpoints (hashKey) VALUES " . implode(", ", $joinList); } + /** + * @param OutPointSerializer $serializer + * @param array $deleteOutPoints + * @param array $newUtxos + * @param array $specificDeletes + */ + public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []) + { + $deleteUtxos = false; + if (!$deleteUtxos && count($deleteOutPoints) > 0) { + $deleteUtxos = true; + } + + //if (true === $deleteUtxos) { + //} + + if (count($newUtxos) > 0) { + $utxoQuery = []; + $utxoValues = []; + foreach ($newUtxos as $c => $utxo) { + $utxoQuery[] = "(:hash$c, :v$c, :s$c)"; + $utxoValues["hash$c"] = $serializer->serialize($utxo->getOutPoint())->getBinary(); + $utxoValues["v$c"] = $utxo->getOutput()->getValue(); + $utxoValues["s$c"] = $utxo->getOutput()->getScript()->getBinary(); + } + + $insertUtxos = $this->dbh->prepare('INSERT INTO utxo (hashKey, value, scriptPubKey) VALUES ' . implode(', ', $utxoQuery)); + $insertUtxos->execute($utxoValues); + } + } + + public function deleteUtxoView() + { + $delete = $this->dbh->prepare("DROP VIEW IF EXISTS utxo_view "); + $delete->execute(); + } + + /** + * @param OutPointSerializer $outpointSerializer + * @param OutPointInterface[] $outpoints + * @return \BitWasp\Bitcoin\Utxo\Utxo[] + */ + public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints) + { + $requiredCount = count($outpoints); + $t1 = microtime(true); + + $joinList = []; + $queryValues = []; + foreach ($outpoints as $i => $outpoint) { + $queryValues[] = $outpointSerializer->serialize($outpoint)->getBinary(); + $joinList[] = "?"; + } + $this->deleteUtxoView(); + + $sql = "create view utxo_view as select * from utxo where hashKey in (".implode(",", $joinList).")"; + $prepared = $this->dbh->prepare($sql); + $prepared->execute($queryValues); + + $select = $this->dbh->prepare("SELECT * FROM utxo_view"); + $select->execute(); + + $outputSet = []; + foreach ($select->fetchAll(\PDO::FETCH_ASSOC) as $utxo) { + $outpoint = $outpointSerializer->parse(new Buffer($utxo['hashKey'])); + $outputSet[] = new DbUtxo($utxo['id'], $outpoint, new TransactionOutput($utxo['value'], new Script(new Buffer($utxo['scriptPubKey'])))); + } + + if (count($outputSet) < $requiredCount) { + throw new \RuntimeException('Less than (' . count($outputSet) . ') required amount (' . $requiredCount . ')returned'); + } + + echo "utxos took " . (microtime(true) - $t1) . " seconds\n"; + return $outputSet; + } + /** * @param OutPointSerializer $outpointSerializer * @param OutPointInterface[] $outpoints diff --git a/src/DbInterface.php b/src/DbInterface.php index c446dc1..faea720 100644 --- a/src/DbInterface.php +++ b/src/DbInterface.php @@ -24,7 +24,10 @@ interface DbInterface * @return \PDO */ public function getPdo(); - + + public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints); + public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []); + public function deleteUtxoView(); /** * */ diff --git a/src/DebugDb.php b/src/DebugDb.php index 5bd4dae..9455a1b 100644 --- a/src/DebugDb.php +++ b/src/DebugDb.php @@ -27,6 +27,22 @@ public function __construct(DbInterface $db) { $this->db = $db; } + public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints) + { + echo __FUNCTION__ . PHP_EOL; + return $this->db->createMiniUtxoView($outpointSerializer, $outpoints); + } + public function deleteUtxoView() + { + echo __FUNCTION__ . PHP_EOL; + return $this->db->deleteUtxoView(); + // TODO: Implement deleteUtxoView() method. + } + public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []) + { + echo __FUNCTION__ . PHP_EOL; + return $this->db->updateUtxoView($serializer, $deleteOutPoints, $newUtxos, $specificDeletes); + } public function getPdo() { diff --git a/src/Index/Blocks.php b/src/Index/Blocks.php index f108708..509c6d4 100644 --- a/src/Index/Blocks.php +++ b/src/Index/Blocks.php @@ -297,7 +297,8 @@ public function accept(BlockInterface $block, Headers $headers, $checkSignatures $state->updateLastBlock($index); $this->forks->next($index); - + echo "Finished block\n"; + return $index; } } From aa1024b2104c29c36a194a3c113e50619eb3d13c Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 8 Apr 2016 17:18:25 +0100 Subject: [PATCH 2/7] track time taken for updateUtxoView --- src/Db.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Db.php b/src/Db.php index ed9eed1..dcb154a 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1070,6 +1070,7 @@ private function createInsertJoinSql(OutPointSerializer $serializer, array $outp */ public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []) { + $m1 = microtime(true); $deleteUtxos = false; if (!$deleteUtxos && count($deleteOutPoints) > 0) { $deleteUtxos = true; @@ -1091,6 +1092,7 @@ public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutP $insertUtxos = $this->dbh->prepare('INSERT INTO utxo (hashKey, value, scriptPubKey) VALUES ' . implode(', ', $utxoQuery)); $insertUtxos->execute($utxoValues); } + echo "UpdateUtxoView: " . (microtime(true) - $m1) . " seconds\n"; } public function deleteUtxoView() From 6d1017271642c3457eb54806259aec33be974620 Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 8 Apr 2016 18:38:02 +0100 Subject: [PATCH 3/7] Delete utxo.id column --- src/Chain/CachingUtxoSet.php | 5 ++--- src/Db.php | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Chain/CachingUtxoSet.php b/src/Chain/CachingUtxoSet.php index 2444e57..9df33ee 100644 --- a/src/Chain/CachingUtxoSet.php +++ b/src/Chain/CachingUtxoSet.php @@ -96,8 +96,7 @@ public function fetchView(array $requiredOutpoints) $utxos = []; $required = []; $cacheHits = []; - $a = 0; - $b = 0; + foreach ($requiredOutpoints as $c => $outpoint) { if ($this->useCaching) { $key = $this->outpointSerializer->serialize($outpoint)->getBinary(); @@ -105,7 +104,7 @@ public function fetchView(array $requiredOutpoints) list ($value, $scriptPubKey) = $this->set->fetch($key); $cacheHits[] = $key; $utxos[] = new Utxo($outpoint, new TransactionOutput($value, new Script(new Buffer($scriptPubKey)))); - $a++; + continue; } } diff --git a/src/Db.php b/src/Db.php index dcb154a..8422e5a 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1119,7 +1119,7 @@ public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array } $this->deleteUtxoView(); - $sql = "create view utxo_view as select * from utxo where hashKey in (".implode(",", $joinList).")"; + $sql = "create view utxo_view as select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; $prepared = $this->dbh->prepare($sql); $prepared->execute($queryValues); @@ -1129,7 +1129,7 @@ public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outputSet = []; foreach ($select->fetchAll(\PDO::FETCH_ASSOC) as $utxo) { $outpoint = $outpointSerializer->parse(new Buffer($utxo['hashKey'])); - $outputSet[] = new DbUtxo($utxo['id'], $outpoint, new TransactionOutput($utxo['value'], new Script(new Buffer($utxo['scriptPubKey'])))); + $outputSet[] = new Utxo($outpoint, new TransactionOutput($utxo['value'], new Script(new Buffer($utxo['scriptPubKey'])))); } if (count($outputSet) < $requiredCount) { From f6b8b008463ffc34f580298c2a468a1a148baca2 Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 8 Apr 2016 18:40:47 +0100 Subject: [PATCH 4/7] remove utxo.id from schema --- sql/schema.sql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sql/schema.sql b/sql/schema.sql index b3673d6..07dd724 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -179,7 +179,6 @@ CREATE TABLE IF NOT EXISTS `transaction_output` ( -- CREATE TABLE IF NOT EXISTS `utxo` ( - `id` int(11) NOT NULL, `hashKey` varbinary(36) NOT NULL, `value` bigint(32) NOT NULL, `scriptPubKey` blob NOT NULL @@ -257,7 +256,6 @@ ADD KEY `parent_tx` (`parent_tx`); -- Indexes for table `utxo` -- ALTER TABLE `utxo` -ADD PRIMARY KEY (`id`), ADD KEY `hashKeyIdx` (`hashKey`); -- @@ -304,11 +302,6 @@ MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT; -- ALTER TABLE `transaction_output` MODIFY `id` int(9) NOT NULL AUTO_INCREMENT; --- --- AUTO_INCREMENT for table `utxo` --- -ALTER TABLE `utxo` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; \ No newline at end of file From 92a9d4452b4d1b24b244d3cbc299d6beda20815e Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 22 Apr 2016 17:36:28 +0100 Subject: [PATCH 5/7] Remove usage of mysql views, and reintroduce utxo.id --- sql/schema.sql | 40 ++++++++++++++++++++++++++++-------- src/Chain/CachingUtxoSet.php | 18 ++++++++++------ src/Db.php | 36 +++++++++++++++++++------------- src/DbInterface.php | 2 +- src/DebugDb.php | 4 ++-- src/Index/Blocks.php | 2 +- 6 files changed, 69 insertions(+), 33 deletions(-) diff --git a/sql/schema.sql b/sql/schema.sql index 07dd724..270c22c 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -3,9 +3,9 @@ -- http://www.phpmyadmin.net -- -- Host: localhost --- Generation Time: Apr 05, 2016 at 09:43 PM --- Server version: 5.6.28-0ubuntu0.15.10.1 --- PHP Version: 5.6.11-1ubuntu3.1 +-- Generation Time: Apr 22, 2016 at 04:34 PM +-- Server version: 5.6.30-0ubuntu0.15.10.1 +-- PHP Version: 5.6.11-1ubuntu3.2 SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; SET time_zone = "+00:00"; @@ -39,12 +39,6 @@ CREATE TABLE IF NOT EXISTS `active_fork` ( -- -------------------------------------------------------- -CREATE TABLE IF NOT EXISTS `outpoints` ( - `hashKey` varbinary(36) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -ALTER TABLE `outpoints` -ADD KEY `hkidx` (`hashKey`); -- -- Table structure for table `blockIndex` -- @@ -101,6 +95,16 @@ CREATE TABLE IF NOT EXISTS `iindex` ( -- -------------------------------------------------------- +-- +-- Table structure for table `outpoints` +-- + +CREATE TABLE IF NOT EXISTS `outpoints` ( + `hashKey` varbinary(36) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- -------------------------------------------------------- + -- -- Table structure for table `peers` -- @@ -179,11 +183,17 @@ CREATE TABLE IF NOT EXISTS `transaction_output` ( -- CREATE TABLE IF NOT EXISTS `utxo` ( + `id` int(9) NOT NULL, `hashKey` varbinary(36) NOT NULL, + `height` int(9) NOT NULL, `value` bigint(32) NOT NULL, `scriptPubKey` blob NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +-- -------------------------------------------------------- + +-- -------------------------------------------------------- + -- -- Indexes for dumped tables -- @@ -224,6 +234,12 @@ ALTER TABLE `iindex` ADD UNIQUE KEY `header_id` (`header_id`), ADD KEY `lft` (`lft`,`rgt`); +-- +-- Indexes for table `outpoints` +-- +ALTER TABLE `outpoints` +ADD KEY `o` (`hashKey`); + -- -- Indexes for table `retarget` -- @@ -256,6 +272,7 @@ ADD KEY `parent_tx` (`parent_tx`); -- Indexes for table `utxo` -- ALTER TABLE `utxo` +ADD PRIMARY KEY (`id`), ADD KEY `hashKeyIdx` (`hashKey`); -- @@ -302,6 +319,11 @@ MODIFY `id` bigint(20) NOT NULL AUTO_INCREMENT; -- ALTER TABLE `transaction_output` MODIFY `id` int(9) NOT NULL AUTO_INCREMENT; +-- +-- AUTO_INCREMENT for table `utxo` +-- +ALTER TABLE `utxo` +MODIFY `id` int(9) NOT NULL AUTO_INCREMENT; /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; \ No newline at end of file diff --git a/src/Chain/CachingUtxoSet.php b/src/Chain/CachingUtxoSet.php index 9df33ee..5989f33 100644 --- a/src/Chain/CachingUtxoSet.php +++ b/src/Chain/CachingUtxoSet.php @@ -52,17 +52,23 @@ public function __construct(DbInterface $db) } /** - * @param OutPointInterface[] $deleteOutPoints - * @param Utxo[] $newUtxos + * @param UtxoView $view + * @param array $deleteOutPoints + * @param array $newUtxos */ - public function applyBlock(array $deleteOutPoints, array $newUtxos) + public function applyBlock(UtxoView $view, array $deleteOutPoints, array $newUtxos) { - $this->db->transaction(function () use ($deleteOutPoints, $newUtxos) { + $this->db->transaction(function () use ($view, $deleteOutPoints, $newUtxos) { //if (!empty($this->cacheHits)) { // $this->db->appendUtxoViewKeys($this->cacheHits); // } - - $this->db->updateUtxoView($this->outpointSerializer, $deleteOutPoints, $newUtxos, []); + + $deleteUtxos = []; + foreach ($deleteOutPoints as $outpoint) { + $deleteUtxos[] = $view->fetch($outpoint); + } + + $this->db->updateUtxoView($this->outpointSerializer, $deleteUtxos, $newUtxos, []); //$this->db->updateUtxoSet($this->outpointSerializer, $deleteOutPoints, $newUtxos, $this->cacheHits); }); diff --git a/src/Db.php b/src/Db.php index 8422e5a..ca0c5b9 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1064,20 +1064,27 @@ private function createInsertJoinSql(OutPointSerializer $serializer, array $outp /** * @param OutPointSerializer $serializer - * @param array $deleteOutPoints + * @param DbUtxo[] $deleteUtxos * @param array $newUtxos * @param array $specificDeletes */ - public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []) + public function updateUtxoView(OutPointSerializer $serializer, array $deleteUtxos, array $newUtxos, array $specificDeletes = []) { $m1 = microtime(true); - $deleteUtxos = false; - if (!$deleteUtxos && count($deleteOutPoints) > 0) { - $deleteUtxos = true; + $doDelete = false; + if (!$doDelete && count($deleteUtxos) > 0) { + $doDelete = true; } - //if (true === $deleteUtxos) { - //} + if (true === $doDelete) { + $deleteValues = []; + foreach ($deleteUtxos as $utxo) { + $deleteValues[] = $utxo->getId(); + } + + $deleteQuery = $this->dbh->prepare('DELETE FROM utxo WHERE id IN (' . implode(', ', $deleteValues) . ')'); + $deleteQuery->execute($deleteValues); + } if (count($newUtxos) > 0) { $utxoQuery = []; @@ -1097,7 +1104,7 @@ public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutP public function deleteUtxoView() { - $delete = $this->dbh->prepare("DROP VIEW IF EXISTS utxo_view "); + $delete = $this->dbh->prepare("DROP VIEW IF EXISTS utxo_view "); $delete->execute(); } @@ -1117,19 +1124,20 @@ public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $queryValues[] = $outpointSerializer->serialize($outpoint)->getBinary(); $joinList[] = "?"; } - $this->deleteUtxoView(); + //$this->deleteUtxoView(); - $sql = "create view utxo_view as select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; + //$sql = "create view utxo_view as select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; + $sql = "select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; $prepared = $this->dbh->prepare($sql); $prepared->execute($queryValues); - $select = $this->dbh->prepare("SELECT * FROM utxo_view"); - $select->execute(); + //$select = $this->dbh->prepare("SELECT * FROM utxo_view"); + //$select->execute(); $outputSet = []; - foreach ($select->fetchAll(\PDO::FETCH_ASSOC) as $utxo) { + foreach ($prepared->fetchAll(\PDO::FETCH_ASSOC) as $utxo) { $outpoint = $outpointSerializer->parse(new Buffer($utxo['hashKey'])); - $outputSet[] = new Utxo($outpoint, new TransactionOutput($utxo['value'], new Script(new Buffer($utxo['scriptPubKey'])))); + $outputSet[] = new DbUtxo($utxo['id'], $outpoint, new TransactionOutput($utxo['value'], new Script(new Buffer($utxo['scriptPubKey'])))); } if (count($outputSet) < $requiredCount) { diff --git a/src/DbInterface.php b/src/DbInterface.php index faea720..5c86175 100644 --- a/src/DbInterface.php +++ b/src/DbInterface.php @@ -26,7 +26,7 @@ interface DbInterface public function getPdo(); public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints); - public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []); + public function updateUtxoView(OutPointSerializer $serializer, array $deleteUtxos, array $newUtxos, array $specificDeletes = []); public function deleteUtxoView(); /** * diff --git a/src/DebugDb.php b/src/DebugDb.php index 9455a1b..f1bc85b 100644 --- a/src/DebugDb.php +++ b/src/DebugDb.php @@ -38,10 +38,10 @@ public function deleteUtxoView() return $this->db->deleteUtxoView(); // TODO: Implement deleteUtxoView() method. } - public function updateUtxoView(OutPointSerializer $serializer, array $deleteOutPoints, array $newUtxos, array $specificDeletes = []) + public function updateUtxoView(OutPointSerializer $serializer, array $deleteUtxos, array $newUtxos, array $specificDeletes = []) { echo __FUNCTION__ . PHP_EOL; - return $this->db->updateUtxoView($serializer, $deleteOutPoints, $newUtxos, $specificDeletes); + return $this->db->updateUtxoView($serializer, $deleteUtxos, $newUtxos, $specificDeletes); } public function getPdo() diff --git a/src/Index/Blocks.php b/src/Index/Blocks.php index 509c6d4..6cb5890 100644 --- a/src/Index/Blocks.php +++ b/src/Index/Blocks.php @@ -292,7 +292,7 @@ public function accept(BlockInterface $block, Headers $headers, $checkSignatures echo "Block insert: ".(microtime(true)-$m) . " seconds\n"; if ($this->config->getItem('config', 'index_utxos', true)) { - $this->utxoSet->applyBlock($blockData->requiredOutpoints, $blockData->remainingNew); + $this->utxoSet->applyBlock($blockData->utxoView, $blockData->requiredOutpoints, $blockData->remainingNew); } $state->updateLastBlock($index); From a8f538f403974707add589b8d6911e05b3cf79ec Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 22 Apr 2016 18:13:50 +0100 Subject: [PATCH 6/7] Index (hash, id) and select against IDs --- sql/schema.sql | 16 ++++++++++++++-- src/Db.php | 3 ++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/sql/schema.sql b/sql/schema.sql index 270c22c..114c253 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -3,7 +3,7 @@ -- http://www.phpmyadmin.net -- -- Host: localhost --- Generation Time: Apr 22, 2016 at 04:34 PM +-- Generation Time: Apr 22, 2016 at 05:12 PM -- Server version: 5.6.30-0ubuntu0.15.10.1 -- PHP Version: 5.6.11-1ubuntu3.2 @@ -192,8 +192,20 @@ CREATE TABLE IF NOT EXISTS `utxo` ( -- -------------------------------------------------------- +-- +-- Stand-in structure for view `utxo_view` +-- +CREATE TABLE IF NOT EXISTS `utxo_view` ( + `id` int(9) + ,`hashKey` varbinary(36) + ,`height` int(9) + ,`value` bigint(32) + ,`scriptPubKey` blob +); + -- -------------------------------------------------------- + -- -- Indexes for dumped tables -- @@ -273,7 +285,7 @@ ADD KEY `parent_tx` (`parent_tx`); -- ALTER TABLE `utxo` ADD PRIMARY KEY (`id`), -ADD KEY `hashKeyIdx` (`hashKey`); +ADD KEY `hashKeyIdx` (`hashKey`,`id`) USING BTREE; -- -- AUTO_INCREMENT for dumped tables diff --git a/src/Db.php b/src/Db.php index ca0c5b9..d43211b 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1127,7 +1127,8 @@ public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array //$this->deleteUtxoView(); //$sql = "create view utxo_view as select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; - $sql = "select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; + $sql = "SELECT * FROM utxo WHERE id in (SELECT id from utxo where hashKey in (". implode(",", $joinList) . "))"; + //$sql = "select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; $prepared = $this->dbh->prepare($sql); $prepared->execute($queryValues); From 2c7cfe422143aa704d65d7bf43e1ac8dd9b347ba Mon Sep 17 00:00:00 2001 From: Thomas Kerin Date: Fri, 22 Apr 2016 18:38:07 +0100 Subject: [PATCH 7/7] refactor query --- src/Db.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Db.php b/src/Db.php index d43211b..0f76852 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1127,7 +1127,13 @@ public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array //$this->deleteUtxoView(); //$sql = "create view utxo_view as select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; - $sql = "SELECT * FROM utxo WHERE id in (SELECT id from utxo where hashKey in (". implode(",", $joinList) . "))"; + $sql = " +SELECT u.* FROM utxo u WHERE u.id in ( + SELECT idx.id from utxo idx where idx.hashKey in (". implode(",", $joinList) . ") +) +"; + + $sql = "SELECT u.* from utxo u JOIN utxo idx on (u.id = idx.id) where idx.hashKey in (".implode(",", $joinList).");"; //$sql = "select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; $prepared = $this->dbh->prepare($sql); $prepared->execute($queryValues);