diff --git a/sql/schema.sql b/sql/schema.sql index b3673d6..114c253 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -3,9 +3,9 @@ -- http://www.phpmyadmin.net -- -- Host: localhost --- Generation Time: Apr 05, 2016 at 09:43 PM --- Server version: 5.6.28-0ubuntu0.15.10.1 --- PHP Version: 5.6.11-1ubuntu3.1 +-- Generation Time: Apr 22, 2016 at 05:12 PM +-- Server version: 5.6.30-0ubuntu0.15.10.1 +-- PHP Version: 5.6.11-1ubuntu3.2 SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; SET time_zone = "+00:00"; @@ -39,12 +39,6 @@ CREATE TABLE IF NOT EXISTS `active_fork` ( -- -------------------------------------------------------- -CREATE TABLE IF NOT EXISTS `outpoints` ( - `hashKey` varbinary(36) NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -ALTER TABLE `outpoints` -ADD KEY `hkidx` (`hashKey`); -- -- Table structure for table `blockIndex` -- @@ -101,6 +95,16 @@ CREATE TABLE IF NOT EXISTS `iindex` ( -- -------------------------------------------------------- +-- +-- Table structure for table `outpoints` +-- + +CREATE TABLE IF NOT EXISTS `outpoints` ( + `hashKey` varbinary(36) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +-- -------------------------------------------------------- + -- -- Table structure for table `peers` -- @@ -179,12 +183,29 @@ CREATE TABLE IF NOT EXISTS `transaction_output` ( -- CREATE TABLE IF NOT EXISTS `utxo` ( - `id` int(11) NOT NULL, + `id` int(9) NOT NULL, `hashKey` varbinary(36) NOT NULL, + `height` int(9) NOT NULL, `value` bigint(32) NOT NULL, `scriptPubKey` blob NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1; +-- -------------------------------------------------------- + +-- +-- Stand-in structure for view `utxo_view` +-- +CREATE TABLE IF NOT EXISTS `utxo_view` ( + `id` int(9) + ,`hashKey` varbinary(36) + ,`height` int(9) + ,`value` bigint(32) + ,`scriptPubKey` blob +); + +-- -------------------------------------------------------- + + -- -- Indexes for dumped tables -- @@ -225,6 +246,12 @@ ALTER TABLE `iindex` ADD UNIQUE KEY `header_id` (`header_id`), ADD KEY `lft` (`lft`,`rgt`); +-- +-- Indexes for table `outpoints` +-- +ALTER TABLE `outpoints` +ADD KEY `o` (`hashKey`); + -- -- Indexes for table `retarget` -- @@ -258,7 +285,7 @@ ADD KEY `parent_tx` (`parent_tx`); -- ALTER TABLE `utxo` ADD PRIMARY KEY (`id`), -ADD KEY `hashKeyIdx` (`hashKey`); +ADD KEY `hashKeyIdx` (`hashKey`,`id`) USING BTREE; -- -- AUTO_INCREMENT for dumped tables @@ -308,7 +335,7 @@ MODIFY `id` int(9) NOT NULL AUTO_INCREMENT; -- AUTO_INCREMENT for table `utxo` -- ALTER TABLE `utxo` -MODIFY `id` int(11) NOT NULL AUTO_INCREMENT; +MODIFY `id` int(9) NOT NULL AUTO_INCREMENT; /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; \ No newline at end of file diff --git a/src/Chain/CachingUtxoSet.php b/src/Chain/CachingUtxoSet.php index eac9f22..5989f33 100644 --- a/src/Chain/CachingUtxoSet.php +++ b/src/Chain/CachingUtxoSet.php @@ -30,7 +30,7 @@ class CachingUtxoSet * @var OutPointSerializer */ private $outpointSerializer; - + private $useCaching = false; /** * UtxoSet constructor. * @param DbInterface $db @@ -52,30 +52,39 @@ public function __construct(DbInterface $db) } /** - * @param OutPointInterface[] $deleteOutPoints - * @param Utxo[] $newUtxos + * @param UtxoView $view + * @param array $deleteOutPoints + * @param array $newUtxos */ - public function applyBlock(array $deleteOutPoints, array $newUtxos) + public function applyBlock(UtxoView $view, array $deleteOutPoints, array $newUtxos) { - $this->db->transaction(function () use ($deleteOutPoints, $newUtxos) { + $this->db->transaction(function () use ($view, $deleteOutPoints, $newUtxos) { //if (!empty($this->cacheHits)) { // $this->db->appendUtxoViewKeys($this->cacheHits); // } - $this->db->updateUtxoSet($this->outpointSerializer, $deleteOutPoints, $newUtxos, $this->cacheHits); + $deleteUtxos = []; + foreach ($deleteOutPoints as $outpoint) { + $deleteUtxos[] = $view->fetch($outpoint); + } + + $this->db->updateUtxoView($this->outpointSerializer, $deleteUtxos, $newUtxos, []); + //$this->db->updateUtxoSet($this->outpointSerializer, $deleteOutPoints, $newUtxos, $this->cacheHits); }); - foreach ($this->cacheHits as $key) { - $this->set->delete($key); - } + if ($this->useCaching) { + foreach ($this->cacheHits as $key) { + $this->set->delete($key); + } - foreach ($newUtxos as $c => $utxo) { - $new = $this->outpointSerializer->serialize($utxo->getOutPoint())->getBinary(); - $this->set->save($new, [ - $newUtxos[$c]->getOutput()-> getValue(), - $newUtxos[$c]->getOutput()->getScript()->getBinary(), - ], 500000); + foreach ($newUtxos as $c => $utxo) { + $new = $this->outpointSerializer->serialize($utxo->getOutPoint())->getBinary(); + $this->set->save($new, [ + $newUtxos[$c]->getOutput()-> getValue(), + $newUtxos[$c]->getOutput()->getScript()->getBinary(), + ], 500000); + } } echo "Inserts: " . count($newUtxos). " | Deletes: " . count($deleteOutPoints). " | " . "CacheHits: " . count($this->cacheHits) .PHP_EOL; @@ -93,29 +102,31 @@ public function fetchView(array $requiredOutpoints) $utxos = []; $required = []; $cacheHits = []; - $a = 0; - $b = 0; + foreach ($requiredOutpoints as $c => $outpoint) { - $key = $this->outpointSerializer->serialize($outpoint)->getBinary(); - if ($this->set->contains($key)) { - list ($value, $scriptPubKey) = $this->set->fetch($key); - $cacheHits[] = $key; - $utxos[] = new Utxo($outpoint, new TransactionOutput($value, new Script(new Buffer($scriptPubKey)))); - $a++; - } else { - $required[] = $outpoint; - $b++; + if ($this->useCaching) { + $key = $this->outpointSerializer->serialize($outpoint)->getBinary(); + if ($this->set->contains($key)) { + list ($value, $scriptPubKey) = $this->set->fetch($key); + $cacheHits[] = $key; + $utxos[] = new Utxo($outpoint, new TransactionOutput($value, new Script(new Buffer($scriptPubKey)))); + + continue; + } } + + $required[] = $outpoint; } if (empty($required) === false) { - $utxos = array_merge($utxos, $this->db->fetchUtxoDbList($this->outpointSerializer, $required)); + $utxos = array_merge($utxos, $this->db->createMiniUtxoView($this->outpointSerializer, $required)); } $this->cacheHits = $cacheHits; return $utxos; } catch (\Exception $e) { + echo $e->getMessage() . PHP_EOL . $e->getTraceAsString().PHP_EOL; throw new \RuntimeException('Failed to find UTXOS in set'); } } diff --git a/src/Db.php b/src/Db.php index fc2ff2e..0f76852 100644 --- a/src/Db.php +++ b/src/Db.php @@ -1062,6 +1062,99 @@ private function createInsertJoinSql(OutPointSerializer $serializer, array $outp return "INSERT INTO outpoints (hashKey) VALUES " . implode(", ", $joinList); } + /** + * @param OutPointSerializer $serializer + * @param DbUtxo[] $deleteUtxos + * @param array $newUtxos + * @param array $specificDeletes + */ + public function updateUtxoView(OutPointSerializer $serializer, array $deleteUtxos, array $newUtxos, array $specificDeletes = []) + { + $m1 = microtime(true); + $doDelete = false; + if (!$doDelete && count($deleteUtxos) > 0) { + $doDelete = true; + } + + if (true === $doDelete) { + $deleteValues = []; + foreach ($deleteUtxos as $utxo) { + $deleteValues[] = $utxo->getId(); + } + + $deleteQuery = $this->dbh->prepare('DELETE FROM utxo WHERE id IN (' . implode(', ', $deleteValues) . ')'); + $deleteQuery->execute($deleteValues); + } + + if (count($newUtxos) > 0) { + $utxoQuery = []; + $utxoValues = []; + foreach ($newUtxos as $c => $utxo) { + $utxoQuery[] = "(:hash$c, :v$c, :s$c)"; + $utxoValues["hash$c"] = $serializer->serialize($utxo->getOutPoint())->getBinary(); + $utxoValues["v$c"] = $utxo->getOutput()->getValue(); + $utxoValues["s$c"] = $utxo->getOutput()->getScript()->getBinary(); + } + + $insertUtxos = $this->dbh->prepare('INSERT INTO utxo (hashKey, value, scriptPubKey) VALUES ' . implode(', ', $utxoQuery)); + $insertUtxos->execute($utxoValues); + } + echo "UpdateUtxoView: " . (microtime(true) - $m1) . " seconds\n"; + } + + public function deleteUtxoView() + { + $delete = $this->dbh->prepare("DROP VIEW IF EXISTS utxo_view "); + $delete->execute(); + } + + /** + * @param OutPointSerializer $outpointSerializer + * @param OutPointInterface[] $outpoints + * @return \BitWasp\Bitcoin\Utxo\Utxo[] + */ + public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints) + { + $requiredCount = count($outpoints); + $t1 = microtime(true); + + $joinList = []; + $queryValues = []; + foreach ($outpoints as $i => $outpoint) { + $queryValues[] = $outpointSerializer->serialize($outpoint)->getBinary(); + $joinList[] = "?"; + } + //$this->deleteUtxoView(); + + //$sql = "create view utxo_view as select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; + $sql = " +SELECT u.* FROM utxo u WHERE u.id in ( + SELECT idx.id from utxo idx where idx.hashKey in (". implode(",", $joinList) . ") +) +"; + + $sql = "SELECT u.* from utxo u JOIN utxo idx on (u.id = idx.id) where idx.hashKey in (".implode(",", $joinList).");"; + //$sql = "select * from utxo where hashKey in (" . implode(",", $joinList) . ")"; + $prepared = $this->dbh->prepare($sql); + $prepared->execute($queryValues); + + //$select = $this->dbh->prepare("SELECT * FROM utxo_view"); + //$select->execute(); + + $outputSet = []; + foreach ($prepared->fetchAll(\PDO::FETCH_ASSOC) as $utxo) { + $outpoint = $outpointSerializer->parse(new Buffer($utxo['hashKey'])); + $outputSet[] = new DbUtxo($utxo['id'], $outpoint, new TransactionOutput($utxo['value'], new Script(new Buffer($utxo['scriptPubKey'])))); + } + + if (count($outputSet) < $requiredCount) { + throw new \RuntimeException('Less than (' . count($outputSet) . ') required amount (' . $requiredCount . ')returned'); + } + + echo "utxos took " . (microtime(true) - $t1) . " seconds\n"; + return $outputSet; + } + /** * @param OutPointSerializer $outpointSerializer * @param OutPointInterface[] $outpoints diff --git a/src/DbInterface.php b/src/DbInterface.php index c446dc1..5c86175 100644 --- a/src/DbInterface.php +++ b/src/DbInterface.php @@ -24,7 +24,10 @@ interface DbInterface * @return \PDO */ public function getPdo(); - + + public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints); + public function updateUtxoView(OutPointSerializer $serializer, array $deleteUtxos, array $newUtxos, array $specificDeletes = []); + public function deleteUtxoView(); /** * */ diff --git a/src/DebugDb.php b/src/DebugDb.php index 5bd4dae..f1bc85b 100644 --- a/src/DebugDb.php +++ b/src/DebugDb.php @@ -27,6 +27,22 @@ public function __construct(DbInterface $db) { $this->db = $db; } + public function createMiniUtxoView(OutPointSerializer $outpointSerializer, array $outpoints) + { + echo __FUNCTION__ . PHP_EOL; + return $this->db->createMiniUtxoView($outpointSerializer, $outpoints); + } + public function deleteUtxoView() + { + echo __FUNCTION__ . PHP_EOL; + return $this->db->deleteUtxoView(); + // TODO: Implement deleteUtxoView() method. + } + public function updateUtxoView(OutPointSerializer $serializer, array $deleteUtxos, array $newUtxos, array $specificDeletes = []) + { + echo __FUNCTION__ . PHP_EOL; + return $this->db->updateUtxoView($serializer, $deleteUtxos, $newUtxos, $specificDeletes); + } public function getPdo() { diff --git a/src/Index/Blocks.php b/src/Index/Blocks.php index f108708..6cb5890 100644 --- a/src/Index/Blocks.php +++ b/src/Index/Blocks.php @@ -292,12 +292,13 @@ public function accept(BlockInterface $block, Headers $headers, $checkSignatures echo "Block insert: ".(microtime(true)-$m) . " seconds\n"; if ($this->config->getItem('config', 'index_utxos', true)) { - $this->utxoSet->applyBlock($blockData->requiredOutpoints, $blockData->remainingNew); + $this->utxoSet->applyBlock($blockData->utxoView, $blockData->requiredOutpoints, $blockData->remainingNew); } $state->updateLastBlock($index); $this->forks->next($index); - + echo "Finished block\n"; + return $index; } }