From f6360a1a39b5591681fd73515ffc6698ededfae7 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 Nov 2015 20:50:56 +0100 Subject: [PATCH 001/379] Initial commit --- .gitignore | 11 + LICENSE | 675 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 686 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..37727f91c --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +# Compiled files +*.o +*.so +*.rlib +*.dll + +# Executables +*.exe + +# Generated by Cargo +/target/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..733c07236 --- /dev/null +++ b/LICENSE @@ -0,0 +1,675 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + {one line to give the program's name and a brief idea of what it does.} + Copyright (C) {year} {name of author} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + {project} Copyright (C) {year} {fullname} + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. + From 789bf0bb0153907e2d3598cacc1b481507752e7a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 Nov 2015 20:57:45 +0100 Subject: [PATCH 002/379] Initial commit. --- Cargo.lock | 4 ++++ Cargo.toml | 4 ++++ README.md | 1 + src/lib.rs | 3 +++ 4 files changed, 12 insertions(+) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 src/lib.rs diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000..8f6ccf6a8 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4 @@ +[root] +name = "ethcore-util" +version = "0.1.0" + diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..5c16a030d --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "ethcore-util" +version = "0.1.0" +authors = ["Ethcore "] diff --git a/README.md b/README.md new file mode 100644 index 000000000..744d98922 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# ethcore-util diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 000000000..a93251b65 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,3 @@ +#[test] +fn it_works() { +} From c595348e31c97762877a50230aa6169fe8733ee7 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 Nov 2015 21:08:33 +0100 Subject: [PATCH 003/379] Git ignore update. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 37727f91c..f56ad1659 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,7 @@ # Executables *.exe +Cargo.lock + # Generated by Cargo /target/ From 3ab8da6c651419628820e6f483551a3b3026a27f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 Nov 2015 21:24:00 +0100 Subject: [PATCH 004/379] Metadata --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 5c16a030d..9cba70e84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,7 @@ [package] +description = "Ethcore utility library" +homepage = "ethcore.io" +licence = "GPL-3.0" name = "ethcore-util" version = "0.1.0" authors = ["Ethcore "] From 89bc463b21039205a9f5d2e5ffddc7f930a20731 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 Nov 2015 21:25:07 +0100 Subject: [PATCH 005/379] More metadata --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9cba70e84..ca377dacb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] description = "Ethcore utility library" -homepage = "ethcore.io" -licence = "GPL-3.0" +homepage = "http://ethcore.io" +license = "GPL-3.0" name = "ethcore-util" version = "0.1.0" authors = ["Ethcore "] From 526993d0c3794a44d63ae0a2cc25c91d5542899b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 24 Nov 2015 22:21:31 +0100 Subject: [PATCH 006/379] Logging deps. --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index ca377dacb..f6b0e38a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,3 +5,7 @@ license = "GPL-3.0" name = "ethcore-util" version = "0.1.0" authors = ["Ethcore "] + +[dependencies] +log = "0.3" +env_logger = "0.3" From 3097a47a9fb363fe34b594c84b79a441ddaeacf4 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 02:53:35 +0100 Subject: [PATCH 007/379] rlp init --- Cargo.lock | 62 +++++++++++++ src/bytes.rs | 151 ++++++++++++++++++++++++++++++ src/lib.rs | 3 + src/rlp.rs | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 470 insertions(+) create mode 100644 src/bytes.rs create mode 100644 src/rlp.rs diff --git a/Cargo.lock b/Cargo.lock index 8f6ccf6a8..daa8da209 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,4 +1,66 @@ [root] name = "ethcore-util" version = "0.1.0" +dependencies = [ + "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/src/bytes.rs b/src/bytes.rs new file mode 100644 index 000000000..06735995f --- /dev/null +++ b/src/bytes.rs @@ -0,0 +1,151 @@ +//! To/From Bytes conversation for basic types +//! +//! Types implementing `ToBytes` and `FromBytes` traits +//! can be easily converted to and from bytes +//! +//! # Examples +//! + +use std::fmt; +use std::error::Error as StdError; + +/// TODO: optimise some conversations +pub trait ToBytes { + fn to_bytes(&self) -> Vec; + fn to_bytes_len(&self) -> usize { self.to_bytes().len() } + fn first_byte(&self) -> Option { self.to_bytes().first().map(|&x| { x })} +} + +impl <'a> ToBytes for &'a str { + fn to_bytes(&self) -> Vec { + From::from(*self) + } + + fn to_bytes_len(&self) -> usize { self.len() } +} + +impl ToBytes for String { + fn to_bytes(&self) -> Vec { + let s: &str = self.as_ref(); + From::from(s) + } + + fn to_bytes_len(&self) -> usize { self.len() } +} + +impl ToBytes for u8 { + fn to_bytes(&self) -> Vec { + match *self { + 0 => vec![], + _ => vec![*self] + } + } + + fn to_bytes_len(&self) -> usize { + match *self { + 0 => 0, + _ => 1 + } + } + fn first_byte(&self) -> Option { + match *self { + 0 => None, + _ => Some(*self) + } + } +} + +impl ToBytes for u64 { + fn to_bytes(&self) -> Vec { + let mut res= vec![]; + let count = self.to_bytes_len(); + for i in 0..count { + let j = count - 1 - i; + res.push((*self >> (j * 8)) as u8); + } + res + } + + fn to_bytes_len(&self) -> usize { 8 - self.leading_zeros() as usize / 8 } +} + +macro_rules! impl_map_to_bytes { + ($from: ident, $to: ty) => { + impl ToBytes for $from { + fn to_bytes(&self) -> Vec { (*self as $to).to_bytes() } + fn to_bytes_len(&self) -> usize { (*self as $to).to_bytes_len() } + } + } +} + +impl_map_to_bytes!(usize, u64); +impl_map_to_bytes!(u16, u64); +impl_map_to_bytes!(u32, u64); + +#[derive(Debug, PartialEq, Eq)] +pub enum FromBytesError { + UnexpectedEnd +} + +impl StdError for FromBytesError { + fn description(&self) -> &str { "from_bytes error" } +} + +impl fmt::Display for FromBytesError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +pub type FromBytesResult = Result; + +/// implements "Sized", so the compiler can deducate the size +/// of the return type +pub trait FromBytes: Sized { + fn from_bytes(bytes: &[u8]) -> FromBytesResult; +} + +impl FromBytes for String { + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + Ok(::std::str::from_utf8(bytes).unwrap().to_string()) + } +} + +impl FromBytes for u8 { + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + match bytes.len() { + 0 => Ok(0), + _ => Ok(bytes[0]) + } + } +} + +impl FromBytes for u64 { + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + match bytes.len() { + 0 => Ok(0), + l => { + let mut res = 0u64; + for i in 0..l { + let shift = (l - 1 - i) * 8; + res = res + ((bytes[i] as u64) << shift); + } + Ok(res) + } + } + } +} + +macro_rules! impl_map_from_bytes { + ($from: ident, $to: ident) => { + impl FromBytes for $from { + fn from_bytes(bytes: &[u8]) -> FromBytesResult<$from> { + $to::from_bytes(bytes).map(| x | { x as $from }) + } + } + } +} + +impl_map_from_bytes!(usize, u64); +impl_map_from_bytes!(u16, u64); +impl_map_from_bytes!(u32, u64); diff --git a/src/lib.rs b/src/lib.rs index a93251b65..17262b3ad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,6 @@ +pub mod bytes; +pub mod rlp; + #[test] fn it_works() { } diff --git a/src/rlp.rs b/src/rlp.rs new file mode 100644 index 000000000..20acb8e5f --- /dev/null +++ b/src/rlp.rs @@ -0,0 +1,254 @@ +//! Rlp serialization module + +use std::fmt; +use std::cell::Cell; +use std::error::Error as StdError; +use bytes::{FromBytes, FromBytesError}; + +/// rlp container +#[derive(Debug)] +pub struct Rlp<'a>{ + bytes: &'a [u8], + cache: Cell +} + +/// rlp offset +#[derive(Copy, Clone, Debug)] +struct OffsetCache { + index: usize, + offset: usize +} + +impl OffsetCache { + fn new(index: usize, offset: usize) -> OffsetCache { + OffsetCache { index: index, offset: offset } + } +} + +/// stores basic information about item +struct ItemInfo { + prefix_len: usize, + value_len: usize +} + +impl ItemInfo { + fn new(prefix_len: usize, value_len: usize) -> ItemInfo { + ItemInfo { prefix_len: prefix_len, value_len: value_len } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpIsNotArray, + BadRlp, +} +impl StdError for DecoderError { + fn description(&self) -> &str { "builder error" } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { DecoderError::FromBytesError(err) } +} + +impl <'a>Rlp<'a> { + /// returns new instance of `Rlp` + pub fn new(bytes: &'a[u8]) -> Rlp<'a> { + Rlp { + bytes: bytes, + cache: Cell::new(OffsetCache::new(usize::max_value(), 0)) + } + } + + /// get container subset at given index + /// + /// paren container caches searched position + pub fn at(&self, index: usize) -> Result, DecoderError> { + if !self.is_array() { + return Err(DecoderError::RlpIsNotArray); + } + + // move to cached position if it's index is less or equal to + // current search index, otherwise move to beginning of array + let c = self.cache.get(); + let (mut bytes, to_skip) = match c.index <= index { + true => (try!(Rlp::consume(self.bytes, c.offset)), index - c.index), + false => (try!(self.consume_array_prefix()), index) + }; + + // skip up to x items + bytes = try!(Rlp::consume_items(bytes, to_skip)); + + // update the cache + self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); + + // construct new rlp + let found = try!(Rlp::item_info(bytes)); + Ok(Rlp::new(&bytes[0..found.prefix_len + found.value_len])) + } + + /// returns true if rlp is an array + pub fn is_array(&self) -> bool { + self.bytes.len() > 0 && self.bytes[0] >= 0xc0 + } + + /// returns true if rlp is a value + pub fn is_value(&self) -> bool { + self.bytes.len() > 0 && self.bytes[0] <= 0xbf + } + + /// returns rlp iterator + pub fn iter(&'a self) -> RlpIterator<'a> { + self.into_iter() + } + + /// consumes first found prefix + fn consume_array_prefix(&self) -> Result<&'a [u8], DecoderError> { + let item = try!(Rlp::item_info(self.bytes)); + let bytes = try!(Rlp::consume(self.bytes, item.prefix_len)); + Ok(bytes) + } + + /// consumes fixed number of items + fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { + let mut result = bytes; + for _ in 0..items { + let i = try!(Rlp::item_info(result)); + result = try!(Rlp::consume(result, (i.prefix_len + i.value_len))); + } + Ok(result) + } + + /// return first item info + fn item_info(bytes: &[u8]) -> Result { + let item = match bytes.first().map(|&x| x) { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => ItemInfo::new(0, 1), + Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let prefix_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); + ItemInfo::new(prefix_len, value_len) + } + Some(l @ 0xc0...0xf7) => ItemInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let prefix_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); + ItemInfo::new(prefix_len, value_len) + }, + _ => return Err(DecoderError::BadRlp) + }; + + match item.prefix_len + item.value_len <= bytes.len() { + true => Ok(item), + false => Err(DecoderError::RlpIsTooShort) + } + } + + /// consumes slice prefix of length `len` + fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { + match bytes.len() >= len { + true => Ok(&bytes[len..]), + false => Err(DecoderError::RlpIsTooShort) + } + } +} + +/// non-consuming rlp iterator +pub struct RlpIterator<'a> { + rlp: &'a Rlp<'a>, + index: usize +} + +impl <'a> IntoIterator for &'a Rlp<'a> { + type Item = Rlp<'a>; + type IntoIter = RlpIterator<'a>; + + fn into_iter(self) -> Self::IntoIter { + RlpIterator { rlp: self, index: 0 } + } +} + +impl <'a> Iterator for RlpIterator<'a> { + type Item = Rlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.at(index).ok(); + self.index += 1; + result + } +} + +#[cfg(test)] +mod tests { + use rlp; + use rlp::Rlp; + + #[test] + fn rlp_at() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = Rlp::new(&data); + assert!(rlp.is_array()); + + let cat = rlp.at(0).unwrap(); + assert!(cat.is_value()); + assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + + let dog = rlp.at(1).unwrap(); + assert!(dog.is_value()); + assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_value()); + assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + } + } + + #[test] + fn rlp_at_err() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; + { + let rlp = Rlp::new(&data); + assert!(rlp.is_array()); + + let cat_err = rlp.at(0).unwrap_err(); + assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); + + let dog_err = rlp.at(1).unwrap_err(); + assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); + } + } + + #[test] + fn rlp_iter() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = Rlp::new(&data); + let mut iter = rlp.iter(); + + let cat = iter.next().unwrap(); + assert!(cat.is_value()); + assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + + let dog = iter.next().unwrap(); + assert!(dog.is_value()); + assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_value()); + assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + } + } +} + From a4db001a6a621240210dcbdf0b0aedf9de6af8ac Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 03:01:47 +0100 Subject: [PATCH 008/379] added test for rlp iterator and *.swp to gitignore --- .gitignore | 3 +++ src/rlp.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index f56ad1659..eabd0a44e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ Cargo.lock # Generated by Cargo /target/ + +# Vim +*.swp diff --git a/src/rlp.rs b/src/rlp.rs index 20acb8e5f..2f97390c7 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -245,6 +245,9 @@ mod tests { assert!(dog.is_value()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + let none = iter.next(); + assert!(none.is_none()); + let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_value()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); From 50c9d607789886dc91c1d4c200f08edacfe019a6 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 09:58:24 +0100 Subject: [PATCH 009/379] renamed RlpIsNotArray to RlpExpectedToBeArray --- src/rlp.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 2f97390c7..a0fe455b5 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -41,7 +41,7 @@ impl ItemInfo { pub enum DecoderError { FromBytesError(FromBytesError), RlpIsTooShort, - RlpIsNotArray, + RlpExpectedToBeArray, BadRlp, } impl StdError for DecoderError { @@ -72,7 +72,7 @@ impl <'a>Rlp<'a> { /// paren container caches searched position pub fn at(&self, index: usize) -> Result, DecoderError> { if !self.is_array() { - return Err(DecoderError::RlpIsNotArray); + return Err(DecoderError::RlpExpectedToBeArray); } // move to cached position if it's index is less or equal to From 0eda699aa57df8fd527ea3f276f0ae91922ecd1e Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 12:46:27 +0100 Subject: [PATCH 010/379] rlp encoder --- src/rlp.rs | 156 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 153 insertions(+), 3 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index a0fe455b5..f8422ed7e 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -2,8 +2,10 @@ use std::fmt; use std::cell::Cell; +use std::io::{Write, BufWriter}; +use std::io::Error as IoError; use std::error::Error as StdError; -use bytes::{FromBytes, FromBytesError}; +use bytes::{ToBytes, FromBytes, FromBytesError}; /// rlp container #[derive(Debug)] @@ -26,13 +28,13 @@ impl OffsetCache { } /// stores basic information about item -struct ItemInfo { +pub struct ItemInfo { prefix_len: usize, value_len: usize } impl ItemInfo { - fn new(prefix_len: usize, value_len: usize) -> ItemInfo { + pub fn new(prefix_len: usize, value_len: usize) -> ItemInfo { ItemInfo { prefix_len: prefix_len, value_len: value_len } } } @@ -189,6 +191,154 @@ impl <'a> Iterator for RlpIterator<'a> { } } +/// shortcut function to encode a `T: Encodable` into a Rlp `Vec` +pub fn encode(object: &E) -> Result, EncoderError> where E: Encodable { + let mut ret: Vec = vec![]; + { + let mut encoder = BasicEncoder::new(&mut ret); + try!(object.encode(&mut encoder)); + } + Ok(ret) +} + +#[derive(Debug)] +pub enum EncoderError { + IoError(IoError) +} + +impl StdError for EncoderError { + fn description(&self) -> &str { "encoder error" } +} + +impl fmt::Display for EncoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for EncoderError { + fn from(err: IoError) -> EncoderError { EncoderError::IoError(err) } +} + +pub trait Encodable { + fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder; + fn item_info(&self) -> ItemInfo; +} + +pub trait Encoder { + type Error; + + fn emit_value(&mut self, value: &V) -> Result<(), Self::Error> where V: Encodable + ToBytes; + fn emit_array(&mut self, array: &[V]) -> Result<(), Self::Error> where V: Encodable; +} + +impl Encodable for T where T: ToBytes { + fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { + encoder.emit_value(self) + } + + fn item_info(&self) -> ItemInfo { + match self.to_bytes_len() { + // just 0 + 0 => ItemInfo::new(0, 1), + // byte is its own encoding + 1 if self.first_byte().unwrap() < 0x80 => ItemInfo::new(0, 1), + // (prefix + length), followed by the stirng + len @ 1...55 => ItemInfo::new(1, len), + // (prefix + length of length), followed by the length, followed by the value + len => ItemInfo::new(1 + len.to_bytes_len(), len) + } + } +} + +impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { + fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { + encoder.emit_array(self) + } + + fn item_info(&self) -> ItemInfo { + let prefix_len = match self.len() { + 0...55 => 1, + len => len.to_bytes_len() + }; + + let value_len = self.iter().fold(0, |acc, ref enc| { + let item = enc.item_info(); + acc + item.prefix_len + item.value_len + }); + + ItemInfo::new(prefix_len, value_len) + } +} + +impl Encodable for Vec where T: Encodable { + fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { + let r: &[T] = self.as_ref(); + r.encode(encoder) + } + + fn item_info(&self) -> ItemInfo { + let r: &[T] = self.as_ref(); + r.item_info() + } +} + +struct BasicEncoder where W: Write { + writer: BufWriter +} + +impl BasicEncoder where W: Write { + pub fn new(writer: W) -> BasicEncoder { + BasicEncoder { writer: BufWriter::new(writer) } + } +} + +impl Encoder for BasicEncoder where W: Write { + type Error = EncoderError; + + fn emit_value(&mut self, value: &V) -> Result<(), Self::Error> where V: Encodable + ToBytes { + let v = value.to_bytes(); + let bytes: &[u8] = v.as_ref(); + + match bytes.len() { + // just 0 + 0 => { try!(self.writer.write(&[0x80u8])); }, + // byte is its own encoding + 1 if bytes[0] < 0x80 => { try!(self.writer.write(bytes)); }, + // (prefix + length), followed by the string + len @ 1 ... 55 => { + try!(self.writer.write(&[0x80u8 + len as u8])); + try!(self.writer.write(bytes)); + } + // (prefix + length of length), followed by the length, followd by the string + len => { + try!(self.writer.write(&[0xb7 + len.to_bytes_len() as u8])); + try!(self.writer.write(&len.to_bytes())); + try!(self.writer.write(bytes)); + } + } + Ok(()) + } + + fn emit_array(&mut self, array: &[V]) -> Result<(), Self::Error> where V: Encodable { + let item = array.item_info(); + + match item.value_len { + len @ 0...55 => { try!(self.writer.write(&[0xc0u8 + len as u8])); } + len => { + try!(self.writer.write(&[0x7fu8 + len.to_bytes_len() as u8])); + try!(self.writer.write(&len.to_bytes())); + } + }; + + for el in array.iter() { + try!(el.encode(self)); + } + + Ok(()) + } +} + #[cfg(test)] mod tests { use rlp; From 6006f7cc38546b9e568f425fc71e5b01144e03ea Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 13:13:26 +0100 Subject: [PATCH 011/379] rlp encoding tests --- src/rlp.rs | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index f8422ed7e..c2e5c6ff4 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -403,5 +403,113 @@ mod tests { assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); } } + + struct ETestPair(T, Vec) where T: rlp::Encodable; + + fn run_encode_tests(tests: Vec>) where T: rlp::Encodable { + for t in &tests { + let res = rlp::encode(&t.0).unwrap(); + assert_eq!(res, &t.1[..]); + } + } + + #[test] + fn encode_u8() { + let tests = vec![ + ETestPair(0u8, vec![0x80u8]), + ETestPair(15, vec![15]), + ETestPair(55, vec![55]), + ETestPair(56, vec![56]), + ETestPair(0x7f, vec![0x7f]), + ETestPair(0x80, vec![0x81, 0x80]), + ETestPair(0xff, vec![0x81, 0xff]), + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_u16() { + let tests = vec![ + ETestPair(0u16, vec![0x80u8]), + ETestPair(0x100, vec![0x82, 0x01, 0x00]), + ETestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_u32() { + let tests = vec![ + ETestPair(0u32, vec![0x80u8]), + ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_u64() { + let tests = vec![ + ETestPair(0u64, vec![0x80u8]), + ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_str() { + let tests = vec![ + ETestPair("cat", vec![0x83, b'c', b'a', b't']), + ETestPair("dog", vec![0x83, b'd', b'o', b'g']), + ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), + ETestPair("", vec![0x80]), + ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', b'r', + b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', b't', b',', + b' ', b'c', b'o', b'n', b's', b'e', b'c', b't', b'e', b't', + b'u', b'r', b' ', b'a', b'd', b'i', b'p', b'i', b's', b'i', + b'c', b'i', b'n', b'g', b' ', b'e', b'l', b'i', b't']) + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_vector_u8() { + let tests = vec![ + ETestPair(vec![], vec![0xc0]), + ETestPair(vec![15u8], vec![0xc1, 0x0f]), + ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_vector_u64() { + let tests = vec![ + ETestPair(vec![], vec![0xc0]), + ETestPair(vec![15u64], vec![0xc1, 0x0f]), + ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_vector_str() { + let tests = vec![ + ETestPair(vec!["cat", "dog"], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']) + ]; + run_encode_tests(tests); + } + + #[test] + fn encode_vector_of_vectors_str() { + let tests = vec![ + ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't']) + ]; + run_encode_tests(tests); + } } From 798b8b0fb884c4a94bd69fd614770da43470663a Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 17:54:50 +0100 Subject: [PATCH 012/379] rlp stream --- src/rlp.rs | 103 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index c2e5c6ff4..2e8d52902 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -191,8 +191,86 @@ impl <'a> Iterator for RlpIterator<'a> { } } +/// container that should be used to encoding the rlp +pub struct RlpStream { + len: usize, + max_len: usize, + bytes: Vec, + last_err: Option +} + +impl RlpStream { + /// create new container of size `max_len` + pub fn new(max_len: usize) -> RlpStream { + RlpStream { + len: 0, + max_len: max_len, + bytes: vec![], + last_err: None + } + } + + /// apends value to the end of stream, chainable + pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { + // if there was an error, stop appending + if !self.last_err.is_none() { + return self + } + + // encode given value and add it at the end of the stream + match encode(object) { + Err(e) => { + self.last_err = Some(e); + return self; + }, + Ok(ref mut v) => { + self.bytes.append(v); + self.len += 1; + }, + }; + + // if array is finished, prepend the length + if self.is_finished() { + self.prepend_the_length(); + } + + // allow chaining calls + self + } + + /// return try if stream is ready + pub fn is_finished(&self) -> bool { + self.len == self.max_len + } + + /// streams out encoded bytes + pub fn out(self) -> Result, EncoderError> { + match self.last_err { + None if self.is_finished() => Ok(self.bytes), + Some(e) => Err(e), + _ => Err(EncoderError::StreamIsUnfinished) + } + } + + /// prepend the length of the bytes to the beginning of the vector + fn prepend_the_length(&mut self) -> () { + let mut v = match self.bytes.len() { + len @ 0...55 => vec![0xc0u8 + len as u8], + len => { + let mut res = vec![0x7fu8 + len.to_bytes_len() as u8]; + let mut b = len.to_bytes(); + res.append(&mut b); + res + } + }; + + v.append(&mut self.bytes); + self.bytes = v; + } +} + /// shortcut function to encode a `T: Encodable` into a Rlp `Vec` -pub fn encode(object: &E) -> Result, EncoderError> where E: Encodable { +fn encode(object: &E) -> Result, EncoderError> where E: Encodable { let mut ret: Vec = vec![]; { let mut encoder = BasicEncoder::new(&mut ret); @@ -203,7 +281,8 @@ pub fn encode(object: &E) -> Result, EncoderError> where E: Encodable #[derive(Debug)] pub enum EncoderError { - IoError(IoError) + IoError(IoError), + StreamIsUnfinished } impl StdError for EncoderError { @@ -257,16 +336,16 @@ impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { } fn item_info(&self) -> ItemInfo { - let prefix_len = match self.len() { - 0...55 => 1, - len => len.to_bytes_len() - }; - let value_len = self.iter().fold(0, |acc, ref enc| { let item = enc.item_info(); acc + item.prefix_len + item.value_len }); + let prefix_len = match value_len { + 0...55 => 1, + len => len.to_bytes_len() + }; + ItemInfo::new(prefix_len, value_len) } } @@ -342,7 +421,7 @@ impl Encoder for BasicEncoder where W: Write { #[cfg(test)] mod tests { use rlp; - use rlp::Rlp; + use rlp::{Rlp, RlpStream}; #[test] fn rlp_at() { @@ -511,5 +590,13 @@ mod tests { ]; run_encode_tests(tests); } + + #[test] + fn rlp_stream() { + let mut stream = RlpStream::new(2); + stream.append(&"cat").append(&"dog"); + let out = stream.out().unwrap(); + assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + } } From fca45496f42f248113a6c522336603dc5a70bb9b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 25 Nov 2015 19:26:40 +0100 Subject: [PATCH 013/379] Add hash and error. --- Cargo.lock | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 + src/error.rs | 13 +++++++ src/hash.rs | 36 +++++++++++++++++ src/lib.rs | 7 ++-- 5 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 src/error.rs create mode 100644 src/hash.rs diff --git a/Cargo.lock b/Cargo.lock index 8f6ccf6a8..223c676f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,4 +1,111 @@ [root] name = "ethcore-util" version = "0.1.0" +dependencies = [ + "arrayvec 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayvec" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "odds 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nodrop" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "odds 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "odds" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unreachable 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-serialize" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unreachable" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "void 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "void" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/Cargo.toml b/Cargo.toml index f6b0e38a4..0b7032bbb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,5 @@ authors = ["Ethcore "] [dependencies] log = "0.3" env_logger = "0.3" +rustc-serialize = "0.3" +arrayvec = "0.3" \ No newline at end of file diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 000000000..9dc471f67 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,13 @@ +use rustc_serialize::hex::*; + +#[derive(Debug)] +pub enum EthcoreError { + FromHex(FromHexError), + BadSize +} + +impl From for EthcoreError { + fn from(err: FromHexError) -> EthcoreError { + EthcoreError::FromHex(err) + } +} diff --git a/src/hash.rs b/src/hash.rs new file mode 100644 index 000000000..295da3900 --- /dev/null +++ b/src/hash.rs @@ -0,0 +1,36 @@ +use rustc_serialize::hex::*; +use error::EthcoreError; +use std::str::FromStr; + +macro_rules! impl_hash { + ($from: ident, $size: expr) => { + #[derive(PartialEq, Debug)] + struct $from ([u8; $size]); + + impl FromStr for $from { +// type Output = $from; + type Err = EthcoreError; + + fn from_str(s: &str) -> Result<$from, EthcoreError> { + let a = try!(s.from_hex()); + if a.len() != $size { return Err(EthcoreError::BadSize); } + let mut ret = $from([0;$size]); + for i in 0..$size { + ret.0[i] = a[i]; + } + Ok(ret) + } + } + } +} + +impl_hash!(Hash64, 8); +impl_hash!(Hash128, 16); +impl_hash!(Address, 20); +impl_hash!(Hash256, 32); +//impl_hash!(Hash512, 64); + +#[test] +fn it_works() { + assert_eq!(Hash64::from_str("0123456789abcdef").unwrap(), Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef])); +} diff --git a/src/lib.rs b/src/lib.rs index a93251b65..9aae65008 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ -#[test] -fn it_works() { -} +extern crate rustc_serialize; + +pub mod error; +pub mod hash; \ No newline at end of file From 9f938fbaab03163c9369c5ab3be52a845efe9d4f Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 19:36:27 +0100 Subject: [PATCH 014/379] RlpStream array method --- src/rlp.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 2e8d52902..aa3d08fe4 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -191,7 +191,7 @@ impl <'a> Iterator for RlpIterator<'a> { } } -/// container that should be used to encoding the rlp +/// container that should be used to encode rlp pub struct RlpStream { len: usize, max_len: usize, @@ -200,8 +200,14 @@ pub struct RlpStream { } impl RlpStream { - /// create new container of size `max_len` - pub fn new(max_len: usize) -> RlpStream { + /// create new container for values appended one after another, + /// but not being part of the same array + pub fn new() -> RlpStream { + RlpStream::array(0) + } + + /// create new container for array of size `max_len` + pub fn array(max_len: usize) -> RlpStream { RlpStream { len: 0, max_len: max_len, @@ -238,7 +244,7 @@ impl RlpStream { self } - /// return try if stream is ready + /// return true if stream is ready pub fn is_finished(&self) -> bool { self.len == self.max_len } @@ -593,7 +599,7 @@ mod tests { #[test] fn rlp_stream() { - let mut stream = RlpStream::new(2); + let mut stream = RlpStream::array(2); stream.append(&"cat").append(&"dog"); let out = stream.out().unwrap(); assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); From c5163b796fd344e04900604facf761dcd197b903 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 21:04:43 +0100 Subject: [PATCH 015/379] improved encoder --- src/rlp.rs | 125 +++++++++++++++++++++++++---------------------------- 1 file changed, 58 insertions(+), 67 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index aa3d08fe4..26a300db8 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -2,7 +2,7 @@ use std::fmt; use std::cell::Cell; -use std::io::{Write, BufWriter}; +use std::io::{Write}; use std::io::Error as IoError; use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; @@ -28,7 +28,7 @@ impl OffsetCache { } /// stores basic information about item -pub struct ItemInfo { +struct ItemInfo { prefix_len: usize, value_len: usize } @@ -277,12 +277,9 @@ impl RlpStream { /// shortcut function to encode a `T: Encodable` into a Rlp `Vec` fn encode(object: &E) -> Result, EncoderError> where E: Encodable { - let mut ret: Vec = vec![]; - { - let mut encoder = BasicEncoder::new(&mut ret); - try!(object.encode(&mut encoder)); - } - Ok(ret) + let mut encoder = BasicEncoder::new(); + try!(object.encode(&mut encoder)); + Ok(encoder.out()) } #[derive(Debug)] @@ -307,7 +304,6 @@ impl From for EncoderError { pub trait Encodable { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder; - fn item_info(&self) -> ItemInfo; } pub trait Encoder { @@ -321,39 +317,12 @@ impl Encodable for T where T: ToBytes { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { encoder.emit_value(self) } - - fn item_info(&self) -> ItemInfo { - match self.to_bytes_len() { - // just 0 - 0 => ItemInfo::new(0, 1), - // byte is its own encoding - 1 if self.first_byte().unwrap() < 0x80 => ItemInfo::new(0, 1), - // (prefix + length), followed by the stirng - len @ 1...55 => ItemInfo::new(1, len), - // (prefix + length of length), followed by the length, followed by the value - len => ItemInfo::new(1 + len.to_bytes_len(), len) - } - } } impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { encoder.emit_array(self) } - - fn item_info(&self) -> ItemInfo { - let value_len = self.iter().fold(0, |acc, ref enc| { - let item = enc.item_info(); - acc + item.prefix_len + item.value_len - }); - - let prefix_len = match value_len { - 0...55 => 1, - len => len.to_bytes_len() - }; - - ItemInfo::new(prefix_len, value_len) - } } impl Encodable for Vec where T: Encodable { @@ -361,24 +330,46 @@ impl Encodable for Vec where T: Encodable { let r: &[T] = self.as_ref(); r.encode(encoder) } +} - fn item_info(&self) -> ItemInfo { - let r: &[T] = self.as_ref(); - r.item_info() +struct BasicEncoder { + bytes: Vec +} + +impl BasicEncoder { + fn new() -> BasicEncoder { + BasicEncoder { bytes: vec![] } + } + + /// inserts array prefix at given position + fn insert_array_len_at_pos(&mut self, len: usize, pos: usize) -> Result<(), EncoderError> { + // new bytes + let mut res: Vec = vec![]; + { + let (before_slice, after_slice) = self.bytes.split_at(pos); + try!(res.write(before_slice)); + + match len { + 0...55 => { try!(res.write(&[0xc0u8 + len as u8])); } + _ => { + try!(res.write(&[0x7fu8 + len.to_bytes_len() as u8])); + try!(res.write(&len.to_bytes())); + } + }; + + try!(res.write(after_slice)); + } + self.bytes = res; + Ok(()) + } + + /// get encoded value + fn out(self) -> Vec { + self.bytes } } -struct BasicEncoder where W: Write { - writer: BufWriter -} - -impl BasicEncoder where W: Write { - pub fn new(writer: W) -> BasicEncoder { - BasicEncoder { writer: BufWriter::new(writer) } - } -} - -impl Encoder for BasicEncoder where W: Write { +impl Encoder for BasicEncoder { type Error = EncoderError; fn emit_value(&mut self, value: &V) -> Result<(), Self::Error> where V: Encodable + ToBytes { @@ -387,40 +378,40 @@ impl Encoder for BasicEncoder where W: Write { match bytes.len() { // just 0 - 0 => { try!(self.writer.write(&[0x80u8])); }, + 0 => { try!(self.bytes.write(&[0x80u8])); }, // byte is its own encoding - 1 if bytes[0] < 0x80 => { try!(self.writer.write(bytes)); }, + 1 if bytes[0] < 0x80 => { try!(self.bytes.write(bytes)); }, // (prefix + length), followed by the string len @ 1 ... 55 => { - try!(self.writer.write(&[0x80u8 + len as u8])); - try!(self.writer.write(bytes)); + try!(self.bytes.write(&[0x80u8 + len as u8])); + try!(self.bytes.write(bytes)); } // (prefix + length of length), followed by the length, followd by the string len => { - try!(self.writer.write(&[0xb7 + len.to_bytes_len() as u8])); - try!(self.writer.write(&len.to_bytes())); - try!(self.writer.write(bytes)); + try!(self.bytes.write(&[0xb7 + len.to_bytes_len() as u8])); + try!(self.bytes.write(&len.to_bytes())); + try!(self.bytes.write(bytes)); } } Ok(()) } fn emit_array(&mut self, array: &[V]) -> Result<(), Self::Error> where V: Encodable { - let item = array.item_info(); - - match item.value_len { - len @ 0...55 => { try!(self.writer.write(&[0xc0u8 + len as u8])); } - len => { - try!(self.writer.write(&[0x7fu8 + len.to_bytes_len() as u8])); - try!(self.writer.write(&len.to_bytes())); - } - }; + // get len before inserting an array + let before_len = self.bytes.len(); + + // insert all array elements for el in array.iter() { try!(el.encode(self)); } - Ok(()) + // get len after inserting an array + let after_len = self.bytes.len(); + + // diff is array len + let array_len = after_len - before_len; + self.insert_array_len_at_pos(array_len, before_len) } } From c47bf0ce5f9fdfee20fbc9ccf4751215381619df Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 21:14:53 +0100 Subject: [PATCH 016/379] emit_value && emit_array do not have constrains regarding type --- src/rlp.rs | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 26a300db8..e751b9d56 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -309,19 +309,26 @@ pub trait Encodable { pub trait Encoder { type Error; - fn emit_value(&mut self, value: &V) -> Result<(), Self::Error> where V: Encodable + ToBytes; - fn emit_array(&mut self, array: &[V]) -> Result<(), Self::Error> where V: Encodable; + fn emit_value(&mut self, bytes: &[u8]) -> Result<(), Self::Error>; + fn emit_array(&mut self, f: F) -> Result<(), Self::Error> where + F: FnOnce(&mut Self) -> Result<(), Self::Error>; } impl Encodable for T where T: ToBytes { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { - encoder.emit_value(self) + encoder.emit_value(&self.to_bytes()) } } impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { - encoder.emit_array(self) + encoder.emit_array(|e| { + // insert all array elements + for el in self.iter() { + try!(el.encode(e)); + } + Ok(()) + }) } } @@ -372,10 +379,7 @@ impl BasicEncoder { impl Encoder for BasicEncoder { type Error = EncoderError; - fn emit_value(&mut self, value: &V) -> Result<(), Self::Error> where V: Encodable + ToBytes { - let v = value.to_bytes(); - let bytes: &[u8] = v.as_ref(); - + fn emit_value(&mut self, bytes: &[u8]) -> Result<(), Self::Error> { match bytes.len() { // just 0 0 => { try!(self.bytes.write(&[0x80u8])); }, @@ -396,15 +400,14 @@ impl Encoder for BasicEncoder { Ok(()) } - fn emit_array(&mut self, array: &[V]) -> Result<(), Self::Error> where V: Encodable { + fn emit_array(&mut self, f: F) -> Result<(), Self::Error> where + F: FnOnce(&mut Self) -> Result<(), Self::Error> { // get len before inserting an array let before_len = self.bytes.len(); // insert all array elements - for el in array.iter() { - try!(el.encode(self)); - } + try!(f(self)); // get len after inserting an array let after_len = self.bytes.len(); From 94ecd8c3dffdb173adbe87c431df2e57ce9dc455 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 21:16:24 +0100 Subject: [PATCH 017/379] there is no need for item info new to be public --- src/rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp.rs b/src/rlp.rs index e751b9d56..452f71c57 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -34,7 +34,7 @@ struct ItemInfo { } impl ItemInfo { - pub fn new(prefix_len: usize, value_len: usize) -> ItemInfo { + fn new(prefix_len: usize, value_len: usize) -> ItemInfo { ItemInfo { prefix_len: prefix_len, value_len: value_len } } } From 93aeec19ec0694b0812581f2e3de543a3a4710fc Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 21:58:15 +0100 Subject: [PATCH 018/379] array -> list, RlpStream::array -> RlpStream::new_list --- src/rlp.rs | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 452f71c57..be67e4ace 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -43,7 +43,7 @@ impl ItemInfo { pub enum DecoderError { FromBytesError(FromBytesError), RlpIsTooShort, - RlpExpectedToBeArray, + RlpExpectedToBeList, BadRlp, } impl StdError for DecoderError { @@ -73,16 +73,16 @@ impl <'a>Rlp<'a> { /// /// paren container caches searched position pub fn at(&self, index: usize) -> Result, DecoderError> { - if !self.is_array() { - return Err(DecoderError::RlpExpectedToBeArray); + if !self.is_list() { + return Err(DecoderError::RlpExpectedToBeList); } // move to cached position if it's index is less or equal to - // current search index, otherwise move to beginning of array + // current search index, otherwise move to beginning of list let c = self.cache.get(); let (mut bytes, to_skip) = match c.index <= index { true => (try!(Rlp::consume(self.bytes, c.offset)), index - c.index), - false => (try!(self.consume_array_prefix()), index) + false => (try!(self.consume_list_prefix()), index) }; // skip up to x items @@ -96,8 +96,8 @@ impl <'a>Rlp<'a> { Ok(Rlp::new(&bytes[0..found.prefix_len + found.value_len])) } - /// returns true if rlp is an array - pub fn is_array(&self) -> bool { + /// returns true if rlp is an list + pub fn is_list(&self) -> bool { self.bytes.len() > 0 && self.bytes[0] >= 0xc0 } @@ -112,7 +112,7 @@ impl <'a>Rlp<'a> { } /// consumes first found prefix - fn consume_array_prefix(&self) -> Result<&'a [u8], DecoderError> { + fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { let item = try!(Rlp::item_info(self.bytes)); let bytes = try!(Rlp::consume(self.bytes, item.prefix_len)); Ok(bytes) @@ -201,13 +201,13 @@ pub struct RlpStream { impl RlpStream { /// create new container for values appended one after another, - /// but not being part of the same array + /// but not being part of the same list pub fn new() -> RlpStream { - RlpStream::array(0) + RlpStream::new_list(0) } - /// create new container for array of size `max_len` - pub fn array(max_len: usize) -> RlpStream { + /// create new container for list of size `max_len` + pub fn new_list(max_len: usize) -> RlpStream { RlpStream { len: 0, max_len: max_len, @@ -235,7 +235,7 @@ impl RlpStream { }, }; - // if array is finished, prepend the length + // if list is finished, prepend the length if self.is_finished() { self.prepend_the_length(); } @@ -310,7 +310,7 @@ pub trait Encoder { type Error; fn emit_value(&mut self, bytes: &[u8]) -> Result<(), Self::Error>; - fn emit_array(&mut self, f: F) -> Result<(), Self::Error> where + fn emit_list(&mut self, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error>; } @@ -322,8 +322,8 @@ impl Encodable for T where T: ToBytes { impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { - encoder.emit_array(|e| { - // insert all array elements + encoder.emit_list(|e| { + // insert all list elements for el in self.iter() { try!(el.encode(e)); } @@ -348,8 +348,8 @@ impl BasicEncoder { BasicEncoder { bytes: vec![] } } - /// inserts array prefix at given position - fn insert_array_len_at_pos(&mut self, len: usize, pos: usize) -> Result<(), EncoderError> { + /// inserts list prefix at given position + fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> Result<(), EncoderError> { // new bytes let mut res: Vec = vec![]; { @@ -400,21 +400,21 @@ impl Encoder for BasicEncoder { Ok(()) } - fn emit_array(&mut self, f: F) -> Result<(), Self::Error> where + fn emit_list(&mut self, f: F) -> Result<(), Self::Error> where F: FnOnce(&mut Self) -> Result<(), Self::Error> { - // get len before inserting an array + // get len before inserting an list let before_len = self.bytes.len(); - // insert all array elements + // insert all list elements try!(f(self)); - // get len after inserting an array + // get len after inserting an list let after_len = self.bytes.len(); - // diff is array len - let array_len = after_len - before_len; - self.insert_array_len_at_pos(array_len, before_len) + // diff is list len + let list_len = after_len - before_len; + self.insert_list_len_at_pos(list_len, before_len) } } @@ -428,7 +428,7 @@ mod tests { let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; { let rlp = Rlp::new(&data); - assert!(rlp.is_array()); + assert!(rlp.is_list()); let cat = rlp.at(0).unwrap(); assert!(cat.is_value()); @@ -449,7 +449,7 @@ mod tests { let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; { let rlp = Rlp::new(&data); - assert!(rlp.is_array()); + assert!(rlp.is_list()); let cat_err = rlp.at(0).unwrap_err(); assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); @@ -593,7 +593,7 @@ mod tests { #[test] fn rlp_stream() { - let mut stream = RlpStream::array(2); + let mut stream = RlpStream::new_list(2); stream.append(&"cat").append(&"dog"); let out = stream.out().unwrap(); assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); From 9dff89427240e268677f7f8df133d3a90edec3f8 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 22:13:57 +0100 Subject: [PATCH 019/379] do not use io::Write --- src/rlp.rs | 95 +++++++++++++++++++++++------------------------------- 1 file changed, 40 insertions(+), 55 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index be67e4ace..4d146a30d 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -2,8 +2,6 @@ use std::fmt; use std::cell::Cell; -use std::io::{Write}; -use std::io::Error as IoError; use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; @@ -224,16 +222,18 @@ impl RlpStream { } // encode given value and add it at the end of the stream - match encode(object) { - Err(e) => { - self.last_err = Some(e); - return self; - }, - Ok(ref mut v) => { - self.bytes.append(v); - self.len += 1; - }, - }; + self.bytes.extend(encode(object)); + self.len += 1; + + //Err(e) => { + //self.last_err = Some(e); + //return self; + //}, + //Ok(ref mut v) => { + //self.bytes.append(v); + //self.len += 1; + //}, + //}; // if list is finished, prepend the length if self.is_finished() { @@ -276,15 +276,14 @@ impl RlpStream { } /// shortcut function to encode a `T: Encodable` into a Rlp `Vec` -fn encode(object: &E) -> Result, EncoderError> where E: Encodable { +fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); - try!(object.encode(&mut encoder)); - Ok(encoder.out()) + object.encode(&mut encoder); + encoder.out() } #[derive(Debug)] pub enum EncoderError { - IoError(IoError), StreamIsUnfinished } @@ -298,42 +297,34 @@ impl fmt::Display for EncoderError { } } -impl From for EncoderError { - fn from(err: IoError) -> EncoderError { EncoderError::IoError(err) } -} - pub trait Encodable { - fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder; + fn encode(&self, encoder: &mut E) -> () where E: Encoder; } pub trait Encoder { - type Error; - - fn emit_value(&mut self, bytes: &[u8]) -> Result<(), Self::Error>; - fn emit_list(&mut self, f: F) -> Result<(), Self::Error> where - F: FnOnce(&mut Self) -> Result<(), Self::Error>; + fn emit_value(&mut self, bytes: &[u8]) -> (); + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); } impl Encodable for T where T: ToBytes { - fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_value(&self.to_bytes()) } } impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { - fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_list(|e| { // insert all list elements for el in self.iter() { - try!(el.encode(e)); + el.encode(e); } - Ok(()) }) } } impl Encodable for Vec where T: Encodable { - fn encode(&self, encoder: &mut E) -> Result<(), E::Error> where E: Encoder { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { let r: &[T] = self.as_ref(); r.encode(encoder) } @@ -349,25 +340,24 @@ impl BasicEncoder { } /// inserts list prefix at given position - fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> Result<(), EncoderError> { + fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { // new bytes let mut res: Vec = vec![]; { let (before_slice, after_slice) = self.bytes.split_at(pos); - try!(res.write(before_slice)); + res.extend(before_slice); match len { - 0...55 => { try!(res.write(&[0xc0u8 + len as u8])); } + 0...55 => res.push(0xc0u8 + len as u8), _ => { - try!(res.write(&[0x7fu8 + len.to_bytes_len() as u8])); - try!(res.write(&len.to_bytes())); + res.push(0x7fu8 + len.to_bytes_len() as u8); + res.extend(len.to_bytes()); } }; - try!(res.write(after_slice)); + res.extend(after_slice); } self.bytes = res; - Ok(()) } /// get encoded value @@ -377,44 +367,39 @@ impl BasicEncoder { } impl Encoder for BasicEncoder { - type Error = EncoderError; - - fn emit_value(&mut self, bytes: &[u8]) -> Result<(), Self::Error> { + fn emit_value(&mut self, bytes: &[u8]) -> () { match bytes.len() { // just 0 - 0 => { try!(self.bytes.write(&[0x80u8])); }, + 0 => self.bytes.push(0x80u8), // byte is its own encoding - 1 if bytes[0] < 0x80 => { try!(self.bytes.write(bytes)); }, + 1 if bytes[0] < 0x80 => self.bytes.extend(bytes), // (prefix + length), followed by the string len @ 1 ... 55 => { - try!(self.bytes.write(&[0x80u8 + len as u8])); - try!(self.bytes.write(bytes)); + self.bytes.push(0x80u8 + len as u8); + self.bytes.extend(bytes); } // (prefix + length of length), followed by the length, followd by the string len => { - try!(self.bytes.write(&[0xb7 + len.to_bytes_len() as u8])); - try!(self.bytes.write(&len.to_bytes())); - try!(self.bytes.write(bytes)); + self.bytes.push(0xb7 + len.to_bytes_len() as u8); + self.bytes.extend(len.to_bytes()); + self.bytes.extend(bytes); } } - Ok(()) } - fn emit_list(&mut self, f: F) -> Result<(), Self::Error> where - F: FnOnce(&mut Self) -> Result<(), Self::Error> { - + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () { // get len before inserting an list let before_len = self.bytes.len(); // insert all list elements - try!(f(self)); + f(self); // get len after inserting an list let after_len = self.bytes.len(); // diff is list len let list_len = after_len - before_len; - self.insert_list_len_at_pos(list_len, before_len) + self.insert_list_len_at_pos(list_len, before_len); } } @@ -487,7 +472,7 @@ mod tests { fn run_encode_tests(tests: Vec>) where T: rlp::Encodable { for t in &tests { - let res = rlp::encode(&t.0).unwrap(); + let res = rlp::encode(&t.0); assert_eq!(res, &t.1[..]); } } From ebe7c09210e24bda53c7bc560326d505dd33ae4d Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 22:14:43 +0100 Subject: [PATCH 020/379] removed legacy comment --- src/rlp.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 4d146a30d..48dbccc55 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -225,16 +225,6 @@ impl RlpStream { self.bytes.extend(encode(object)); self.len += 1; - //Err(e) => { - //self.last_err = Some(e); - //return self; - //}, - //Ok(ref mut v) => { - //self.bytes.append(v); - //self.len += 1; - //}, - //}; - // if list is finished, prepend the length if self.is_finished() { self.prepend_the_length(); From 1c2c8c4eb5dba77d91c6029c2954e0bad67a253d Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 22:16:50 +0100 Subject: [PATCH 021/379] removed last_err field from RlpStream --- src/rlp.rs | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 48dbccc55..bd5561635 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -193,8 +193,7 @@ impl <'a> Iterator for RlpIterator<'a> { pub struct RlpStream { len: usize, max_len: usize, - bytes: Vec, - last_err: Option + bytes: Vec } impl RlpStream { @@ -209,18 +208,12 @@ impl RlpStream { RlpStream { len: 0, max_len: max_len, - bytes: vec![], - last_err: None + bytes: vec![] } } /// apends value to the end of stream, chainable pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { - // if there was an error, stop appending - if !self.last_err.is_none() { - return self - } - // encode given value and add it at the end of the stream self.bytes.extend(encode(object)); self.len += 1; @@ -241,10 +234,9 @@ impl RlpStream { /// streams out encoded bytes pub fn out(self) -> Result, EncoderError> { - match self.last_err { - None if self.is_finished() => Ok(self.bytes), - Some(e) => Err(e), - _ => Err(EncoderError::StreamIsUnfinished) + match self.is_finished() { + true => Ok(self.bytes), + false => Err(EncoderError::StreamIsUnfinished) } } From 20cadef4f085ec890f8758734298055887733063 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 22:29:21 +0100 Subject: [PATCH 022/379] RlpStream uses BasicEncoder --- src/rlp.rs | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index bd5561635..6161e5cca 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -193,7 +193,7 @@ impl <'a> Iterator for RlpIterator<'a> { pub struct RlpStream { len: usize, max_len: usize, - bytes: Vec + encoder: BasicEncoder } impl RlpStream { @@ -208,14 +208,14 @@ impl RlpStream { RlpStream { len: 0, max_len: max_len, - bytes: vec![] + encoder: BasicEncoder::new() } } /// apends value to the end of stream, chainable pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { // encode given value and add it at the end of the stream - self.bytes.extend(encode(object)); + object.encode(&mut self.encoder); self.len += 1; // if list is finished, prepend the length @@ -235,30 +235,20 @@ impl RlpStream { /// streams out encoded bytes pub fn out(self) -> Result, EncoderError> { match self.is_finished() { - true => Ok(self.bytes), + true => Ok(self.encoder.out()), false => Err(EncoderError::StreamIsUnfinished) } } /// prepend the length of the bytes to the beginning of the vector fn prepend_the_length(&mut self) -> () { - let mut v = match self.bytes.len() { - len @ 0...55 => vec![0xc0u8 + len as u8], - len => { - let mut res = vec![0x7fu8 + len.to_bytes_len() as u8]; - let mut b = len.to_bytes(); - res.append(&mut b); - res - } - }; - - v.append(&mut self.bytes); - self.bytes = v; + let len = self.encoder.bytes.len(); + self.encoder.insert_list_len_at_pos(len, 0); } } /// shortcut function to encode a `T: Encodable` into a Rlp `Vec` -fn encode(object: &E) -> Vec where E: Encodable { +pub fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); object.encode(&mut encoder); encoder.out() From e40bb6d52c3e7e4324b87b2cc711271ccf9208f6 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 25 Nov 2015 23:40:01 +0100 Subject: [PATCH 023/379] rlp append_list --- src/rlp.rs | 90 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 6161e5cca..a085e2fc2 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -2,6 +2,7 @@ use std::fmt; use std::cell::Cell; +use std::collections::LinkedList; use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; @@ -189,10 +190,26 @@ impl <'a> Iterator for RlpIterator<'a> { } } +#[derive(Debug)] +struct ListInfo { + position: usize, + current: usize, + max: usize +} + +impl ListInfo { + fn new(position: usize, max: usize) -> ListInfo { + ListInfo { + position: position, + current: 0, + max: max + } + } +} + /// container that should be used to encode rlp pub struct RlpStream { - len: usize, - max_len: usize, + unfinished_lists: LinkedList, encoder: BasicEncoder } @@ -200,36 +217,51 @@ impl RlpStream { /// create new container for values appended one after another, /// but not being part of the same list pub fn new() -> RlpStream { - RlpStream::new_list(0) + RlpStream { + unfinished_lists: LinkedList::new(), + encoder: BasicEncoder::new() + } } /// create new container for list of size `max_len` - pub fn new_list(max_len: usize) -> RlpStream { - RlpStream { - len: 0, - max_len: max_len, - encoder: BasicEncoder::new() - } + pub fn new_list(len: usize) -> RlpStream { + let mut stream = RlpStream::new(); + stream.append_list(len); + stream } /// apends value to the end of stream, chainable pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { // encode given value and add it at the end of the stream object.encode(&mut self.encoder); - self.len += 1; // if list is finished, prepend the length - if self.is_finished() { - self.prepend_the_length(); + self.try_to_finish(); + + // return chainable self + self + } + + /// declare appending the list of given size + pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { + // push new list + let position = self.encoder.bytes.len(); + match len { + 0 => { + // we may finish, if the appended list len is equal 0 + self.encoder.insert_list_len_at_pos(0, position); + self.try_to_finish(); + }, + _ => self.unfinished_lists.push_back(ListInfo::new(position, len)) } - // allow chaining calls + // return chainable self self } /// return true if stream is ready pub fn is_finished(&self) -> bool { - self.len == self.max_len + self.unfinished_lists.back().is_none() } /// streams out encoded bytes @@ -240,10 +272,22 @@ impl RlpStream { } } - /// prepend the length of the bytes to the beginning of the vector - fn prepend_the_length(&mut self) -> () { - let len = self.encoder.bytes.len(); - self.encoder.insert_list_len_at_pos(len, 0); + /// try to finish lists + fn try_to_finish(&mut self) -> () { + let should_finish = match self.unfinished_lists.back_mut() { + None => false, + Some(ref mut x) => { + x.current += 1; + x.current == x.max + } + }; + + if should_finish { + let x = self.unfinished_lists.pop_back().unwrap(); + let len = self.encoder.bytes.len() - x.position; + self.encoder.insert_list_len_at_pos(len, x.position); + self.try_to_finish(); + } } } @@ -555,5 +599,15 @@ mod tests { let out = stream.out().unwrap(); assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); } + + #[test] + fn rlp_stream_list() { + let mut stream = RlpStream::new_list(3); + stream.append_list(0); + stream.append_list(1).append(&vec![] as &Vec); + stream.append_list(2).append_list(0).append_list(1).append_list(0); + let out = stream.out().unwrap(); + assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); + } } From 31ee96f01c7761e4c9958e94e36719aa2929d6c8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 26 Nov 2015 00:02:43 +0100 Subject: [PATCH 024/379] Display stuff. --- src/hash.rs | 37 ++++++++++++++++++++++++++++++++----- src/lib.rs | 2 ++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 295da3900..94d2af62f 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,14 +1,13 @@ use rustc_serialize::hex::*; use error::EthcoreError; use std::str::FromStr; +use std::fmt; macro_rules! impl_hash { ($from: ident, $size: expr) => { - #[derive(PartialEq, Debug)] - struct $from ([u8; $size]); + pub struct $from (pub [u8; $size]); impl FromStr for $from { -// type Output = $from; type Err = EthcoreError; fn from_str(s: &str) -> Result<$from, EthcoreError> { @@ -21,6 +20,30 @@ macro_rules! impl_hash { Ok(ret) } } + + impl fmt::Debug for $from { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in self.0.iter() { + try!(write!(f, "{:02x}", i)); + } + Ok(()) + } + } + impl fmt::Display for $from { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + (self as &fmt::Debug).fmt(f) + } + } + impl PartialEq for $from { + fn eq(&self, other: &Self) -> bool { + for i in 0..$size { + if self.0[i] != other.0[i] { + return false; + } + } + true + } + } } } @@ -28,9 +51,13 @@ impl_hash!(Hash64, 8); impl_hash!(Hash128, 16); impl_hash!(Address, 20); impl_hash!(Hash256, 32); -//impl_hash!(Hash512, 64); +impl_hash!(Hash512, 64); +impl_hash!(Hash520, 65); +impl_hash!(Hash1024, 128); +impl_hash!(Hash2048, 256); +impl_hash!(Hash4096, 512); #[test] fn it_works() { assert_eq!(Hash64::from_str("0123456789abcdef").unwrap(), Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef])); -} +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 8a65c9873..91c4f8db6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,7 @@ extern crate rustc_serialize; +pub use std::str::FromStr; + pub mod error; pub mod hash; pub mod bytes; From 4748ed522196a66bc67f9a5a0163d7bd2def3065 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 00:09:09 +0100 Subject: [PATCH 025/379] todos --- src/rlp.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index a085e2fc2..3cb240734 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -243,6 +243,7 @@ impl RlpStream { } /// declare appending the list of given size + /// TODO: optimise insertion of empty list pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { // push new list let position = self.encoder.bytes.len(); @@ -356,6 +357,7 @@ impl BasicEncoder { } /// inserts list prefix at given position + /// TODO: optimise it, so it does not copy an array fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { // new bytes let mut res: Vec = vec![]; From 778b143e45838e89df954d77226267f041c99642 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 26 Nov 2015 00:27:20 +0100 Subject: [PATCH 026/379] More tests. --- src/hash.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/hash.rs b/src/hash.rs index 94d2af62f..224bc3699 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -59,5 +59,11 @@ impl_hash!(Hash4096, 512); #[test] fn it_works() { - assert_eq!(Hash64::from_str("0123456789abcdef").unwrap(), Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef])); + let h = Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); + assert_eq!(Hash64::from_str("0123456789abcdef").unwrap(), h); + assert_eq!(format!("{}", h), "0123456789abcdef"); + assert_eq!(format!("{:?}", h), "0123456789abcdef"); + assert!(h == h); + assert!(h != Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); + assert!(h != Hash64([0; 8])); } \ No newline at end of file From 15820887595077fb92350221b5e0a72e7caa510d Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 00:37:53 +0100 Subject: [PATCH 027/379] rlp encoding documentation --- src/rlp.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index 3cb240734..e5ac25799 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1,4 +1,47 @@ //! Rlp serialization module +//! +//! Types implementing `Endocable` and `Decodable` traits +//! can be easily coverted to and from rlp +//! +//! # Examples: +//! +//! ```rust +//! extern crate ethcore_util; +//! use ethcore_util::rlp::{RlpStream}; +//! +//! fn encode_value() { +//! // 1029 +//! let mut stream = RlpStream::new(); +//! stream.append(&1029u32); +//! let out = stream.out().unwrap(); +//! assert_eq!(out, vec![0x82, 0x04, 0x05]); +//! } +//! +//! fn encode_list() { +//! // [ "cat", "dog" ] +//! let mut stream = RlpStream::new_list(2); +//! stream.append(&"cat").append(&"dog"); +//! let out = stream.out().unwrap(); +//! assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); +//! } +//! +//! fn encode_list2() { +//! // [ [], [[]], [ [], [[]] ] ] +//! let mut stream = RlpStream::new_list(3); +//! stream.append_list(0); +//! stream.append_list(1).append_list(0); +//! stream.append_list(2).append_list(0).append_list(1).append_list(0); +//! let out = stream.out().unwrap(); +//! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); +//! } +//! +//! fn main() { +//! encode_value(); +//! encode_list(); +//! encode_list2(); +//! } +//! ``` +//! use std::fmt; use std::cell::Cell; From f5d2d7a2d409e8dbe726db6ed51451b347308a79 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 01:22:33 +0100 Subject: [PATCH 028/379] benchmars. optimised inserting empty list --- benches/rlp.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/rlp.rs | 3 +-- 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 benches/rlp.rs diff --git a/benches/rlp.rs b/benches/rlp.rs new file mode 100644 index 000000000..20ddf25d4 --- /dev/null +++ b/benches/rlp.rs @@ -0,0 +1,46 @@ +//! benchmarking for rlp +//! should be started with: +//! ```bash +//! multirust run nightly cargo bench +//! ``` + +#![feature(test)] + +extern crate test; +extern crate ethcore_util; + +use test::Bencher; +use ethcore_util::rlp::{RlpStream}; + +#[bench] +fn bench_stream_value(b: &mut Bencher) { + b.iter( || { + //1029 + let mut stream = RlpStream::new(); + stream.append(&1029u32); + let _ = stream.out().unwrap(); + }); +} + +#[bench] +fn bench_stream_nested_empty_lists(b: &mut Bencher) { + b.iter( || { + // [ [], [[]], [ [], [[]] ] ] + let mut stream = RlpStream::new_list(3); + stream.append_list(0); + stream.append_list(1).append_list(0); + stream.append_list(2).append_list(0).append_list(1).append_list(0); + let _ = stream.out().unwrap(); + }); +} + +#[bench] +fn bench_stream_1000_empty_lists(b: &mut Bencher) { + b.iter( || { + let mut stream = RlpStream::new(); + for _ in 0..1000 { + stream.append_list(0); + } + let _ = stream.out().unwrap(); + }); +} diff --git a/src/rlp.rs b/src/rlp.rs index e5ac25799..f73417859 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -286,14 +286,13 @@ impl RlpStream { } /// declare appending the list of given size - /// TODO: optimise insertion of empty list pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { // push new list let position = self.encoder.bytes.len(); match len { 0 => { // we may finish, if the appended list len is equal 0 - self.encoder.insert_list_len_at_pos(0, position); + self.encoder.bytes.push(0xc0u8); self.try_to_finish(); }, _ => self.unfinished_lists.push_back(ListInfo::new(position, len)) From e90c07009ec32f5b05ab276f30bd1afcd20dce2a Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 01:26:32 +0100 Subject: [PATCH 029/379] fixed typos in comments --- src/rlp.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index f73417859..a93bb14d3 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -138,7 +138,7 @@ impl <'a>Rlp<'a> { Ok(Rlp::new(&bytes[0..found.prefix_len + found.value_len])) } - /// returns true if rlp is an list + /// returns true if rlp is a list pub fn is_list(&self) -> bool { self.bytes.len() > 0 && self.bytes[0] >= 0xc0 } @@ -448,13 +448,13 @@ impl Encoder for BasicEncoder { } fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () { - // get len before inserting an list + // get len before inserting a list let before_len = self.bytes.len(); // insert all list elements f(self); - // get len after inserting an list + // get len after inserting a list let after_len = self.bytes.len(); // diff is list len From a20d92c03b29c0d9961d797d1dc2ddd18bcc7f5c Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 02:32:50 +0100 Subject: [PATCH 030/379] decoding rlp + tests --- src/rlp.rs | 174 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index a93bb14d3..12d94f4a1 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -86,6 +86,7 @@ pub enum DecoderError { FromBytesError(FromBytesError), RlpIsTooShort, RlpExpectedToBeList, + RlpExpectedToBeValue, BadRlp, } impl StdError for DecoderError { @@ -233,6 +234,61 @@ impl <'a> Iterator for RlpIterator<'a> { } } +/// shortcut function to decode a Rlp `&[u8]` into an object +pub fn decode(bytes: &[u8]) -> Result where T: Decodable { + let rlp = Rlp::new(bytes); + T::decode(&rlp) +} + +pub trait Decodable: Sized { + fn decode(rlp: &Rlp) -> Result; +} + +impl Decodable for T where T: FromBytes { + fn decode(rlp: &Rlp) -> Result { + match rlp.is_value() { + true => BasicDecoder::read_value(rlp.bytes), + false => Err(DecoderError::RlpExpectedToBeValue) + } + } +} + +impl Decodable for Vec where T: Decodable { + fn decode(rlp: &Rlp) -> Result { + match rlp.is_list() { + true => rlp.iter().map(|rlp| T::decode(&rlp)).collect(), + false => Err(DecoderError::RlpExpectedToBeValue) + } + } +} + +trait Decoder { + fn read_value(bytes: &[u8]) -> Result where T: FromBytes; +} + +struct BasicDecoder; + +impl Decoder for BasicDecoder { + fn read_value(bytes: &[u8]) -> Result where T: FromBytes { + match bytes.first().map(|&x| x) { + // rlp is too short + None => Err(DecoderError::RlpIsTooShort), + // single byt value + Some(l @ 0...0x7f) => Ok(try!(T::from_bytes(&[l]))), + // 0-55 bytes + Some(l @ 0x80...0xb7) => Ok(try!(T::from_bytes(&bytes[1..(1 + l as usize - 0x80)]))), + // longer than 55 bytes + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let begin_of_value = 1 as usize + len_of_len; + let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); + Ok(try!(T::from_bytes(&bytes[begin_of_value..begin_of_value + len]))) + }, + _ => Err(DecoderError::BadRlp) + } + } +} + #[derive(Debug)] struct ListInfo { position: usize, @@ -465,8 +521,9 @@ impl Encoder for BasicEncoder { #[cfg(test)] mod tests { + use std::{fmt, cmp}; use rlp; - use rlp::{Rlp, RlpStream}; + use rlp::{Rlp, RlpStream, Decodable}; #[test] fn rlp_at() { @@ -474,18 +531,23 @@ mod tests { { let rlp = Rlp::new(&data); assert!(rlp.is_list()); + let animals = as rlp::Decodable>::decode(&rlp).unwrap(); + assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); assert!(cat.is_value()); assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(String::decode(&cat).unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); assert!(dog.is_value()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + assert_eq!(String::decode(&dog).unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_value()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(String::decode(&cat_again).unwrap(), "cat".to_string()); } } @@ -648,10 +710,118 @@ mod tests { fn rlp_stream_list() { let mut stream = RlpStream::new_list(3); stream.append_list(0); - stream.append_list(1).append(&vec![] as &Vec); + stream.append_list(1).append_list(0); stream.append_list(2).append_list(0).append_list(1).append_list(0); let out = stream.out().unwrap(); assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); } + + struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; + + fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { + for t in &tests { + let res: T = rlp::decode(&t.1).unwrap(); + assert_eq!(res, t.0); + } + } + + #[test] + fn decode_u8() { + let tests = vec![ + DTestPair(0u8, vec![0u8]), + DTestPair(15, vec![15]), + DTestPair(55, vec![55]), + DTestPair(56, vec![56]), + DTestPair(0x7f, vec![0x7f]), + DTestPair(0x80, vec![0x81, 0x80]), + DTestPair(0xff, vec![0x81, 0xff]), + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_u16() { + let tests = vec![ + DTestPair(0u16, vec![0u8]), + DTestPair(0x100, vec![0x82, 0x01, 0x00]), + DTestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_u32() { + let tests = vec![ + DTestPair(0u32, vec![0u8]), + DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_u64() { + let tests = vec![ + DTestPair(0u64, vec![0u8]), + DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_str() { + let tests = vec![ + DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), + DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), + DTestPair("Marek".to_string(), vec![0x85, b'M', b'a', b'r', b'e', b'k']), + DTestPair("".to_string(), vec![0x80]), + DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit".to_string(), + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', b'r', + b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', b't', b',', + b' ', b'c', b'o', b'n', b's', b'e', b'c', b't', b'e', b't', + b'u', b'r', b' ', b'a', b'd', b'i', b'p', b'i', b's', b'i', + b'c', b'i', b'n', b'g', b' ', b'e', b'l', b'i', b't']) + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_vector_u8() { + let tests = vec![ + DTestPair(vec![] as Vec, vec![0xc0]), + DTestPair(vec![15u8], vec![0xc1, 0x0f]), + DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_vector_u64() { + let tests = vec![ + DTestPair(vec![], vec![0xc0]), + DTestPair(vec![15u64], vec![0xc1, 0x0f]), + DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_vector_str() { + let tests = vec![ + DTestPair(vec!["cat".to_string(), "dog".to_string()], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']) + ]; + run_decode_tests(tests); + } + + #[test] + fn decode_vector_of_vectors_str() { + let tests = vec![ + DTestPair(vec![vec!["cat".to_string()]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't']) + ]; + run_decode_tests(tests); + } } From f0a6376a9effd4dbe6a0e06a8ba1c0e9eeb8ef41 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 02:50:21 +0100 Subject: [PATCH 031/379] quick fixes --- benches/rlp.rs | 16 +++++++++++++++- src/rlp.rs | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 20ddf25d4..10397db09 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -10,7 +10,8 @@ extern crate test; extern crate ethcore_util; use test::Bencher; -use ethcore_util::rlp::{RlpStream}; +use ethcore_util::rlp; +use ethcore_util::rlp::{RlpStream, Rlp, Decodable}; #[bench] fn bench_stream_value(b: &mut Bencher) { @@ -34,6 +35,19 @@ fn bench_stream_nested_empty_lists(b: &mut Bencher) { }); } +#[bench] +fn bench_decode_nested_empty_lists(b: &mut Bencher) { + b.iter( || { + // [ [], [[]], [ [], [[]] ] ] + let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; + let rlp = Rlp::new(&data); + let v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); + let v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); + let v2a: Vec = Decodable::decode(&rlp.at(2).unwrap().at(0).unwrap()).unwrap(); + let v2b: Vec> = Decodable::decode(&rlp.at(2).unwrap().at(1).unwrap()).unwrap(); + }); +} + #[bench] fn bench_stream_1000_empty_lists(b: &mut Bencher) { b.iter( || { diff --git a/src/rlp.rs b/src/rlp.rs index 12d94f4a1..e9e175363 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -262,7 +262,7 @@ impl Decodable for Vec where T: Decodable { } } -trait Decoder { +pub trait Decoder { fn read_value(bytes: &[u8]) -> Result where T: FromBytes; } From 71d75765669a0eb21d10b7c0a12a0204cb5953ca Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 12:21:17 +0100 Subject: [PATCH 032/379] fixed return error type --- src/rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp.rs b/src/rlp.rs index e9e175363..cd83a8c7e 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -257,7 +257,7 @@ impl Decodable for Vec where T: Decodable { fn decode(rlp: &Rlp) -> Result { match rlp.is_list() { true => rlp.iter().map(|rlp| T::decode(&rlp)).collect(), - false => Err(DecoderError::RlpExpectedToBeValue) + false => Err(DecoderError::RlpExpectedToBeList) } } } From f95265d6aecfd1a74f567273d786da0735c6065e Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 13:42:42 +0100 Subject: [PATCH 033/379] uint256 --- src/lib.rs | 1 + src/uint.rs | 593 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 594 insertions(+) create mode 100644 src/uint.rs diff --git a/src/lib.rs b/src/lib.rs index 8a65c9873..ab8cac701 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ extern crate rustc_serialize; pub mod error; pub mod hash; +pub mod uint; pub mod bytes; pub mod rlp; diff --git a/src/uint.rs b/src/uint.rs new file mode 100644 index 000000000..149ff5efb --- /dev/null +++ b/src/uint.rs @@ -0,0 +1,593 @@ +// taken from Rust Bitcoin Library (https://github.com/apoelstra/rust-bitcoin) +// original author: Andrew Poelstra + +// Rust Bitcoin Library +// Written in 2014 by +// Andrew Poelstra +// +// To the extent possible under law, the author(s) have dedicated all +// copyright and related and neighboring rights to this software to +// the public domain worldwide. This software is distributed without +// any warranty. +// +// You should have received a copy of the CC0 Public Domain Dedication +// along with this software. +// If not, see . +// + +//! Big unsigned integer types +///! +///! Implementation of a various large-but-fixed sized unsigned integer types. +///! The functions here are designed to be fast. +///! + +use std::fmt; +use std::cmp::{Ord, PartialOrd, Ordering}; +use std::ops::*; +use std::str::FromStr; +use rustc_serialize::hex::{FromHex, FromHexError}; + +macro_rules! impl_map_from { + ($thing:ident, $from:ty, $to:ty) => { + impl From<$from> for $thing { + fn from(value: $from) -> $thing { + From::from(value as $to) + } + } + } +} + +macro_rules! impl_array_newtype { + ($thing:ident, $ty:ty, $len:expr) => { + impl $thing { + #[inline] + /// Converts the object to a raw pointer + pub fn as_ptr(&self) -> *const $ty { + let &$thing(ref dat) = self; + dat.as_ptr() + } + + #[inline] + /// Converts the object to a mutable raw pointer + pub fn as_mut_ptr(&mut self) -> *mut $ty { + let &mut $thing(ref mut dat) = self; + dat.as_mut_ptr() + } + + #[inline] + /// Returns the length of the object as an array + pub fn len(&self) -> usize { $len } + + #[inline] + /// Returns whether the object, as an array, is empty. Always false. + pub fn is_empty(&self) -> bool { false } + } + + impl<'a> From<&'a [$ty]> for $thing { + fn from(data: &'a [$ty]) -> $thing { + assert_eq!(data.len(), $len); + unsafe { + use std::intrinsics::copy_nonoverlapping; + use std::mem; + let mut ret: $thing = mem::uninitialized(); + copy_nonoverlapping(data.as_ptr(), + ret.as_mut_ptr(), + mem::size_of::<$thing>()); + ret + } + } + } + + impl Index for $thing { + type Output = $ty; + + #[inline] + fn index(&self, index: usize) -> &$ty { + let &$thing(ref dat) = self; + &dat[index] + } + } + + impl_index_newtype!($thing, $ty); + + impl PartialEq for $thing { + #[inline] + fn eq(&self, other: &$thing) -> bool { + &self[..] == &other[..] + } + } + + impl Eq for $thing {} + + impl Clone for $thing { + #[inline] + fn clone(&self) -> $thing { + $thing::from(&self[..]) + } + } + + impl Copy for $thing {} + } +} + +macro_rules! impl_index_newtype { + ($thing:ident, $ty:ty) => { + impl Index> for $thing { + type Output = [$ty]; + + #[inline] + fn index(&self, index: Range) -> &[$ty] { + &self.0[index] + } + } + + impl Index> for $thing { + type Output = [$ty]; + + #[inline] + fn index(&self, index: RangeTo) -> &[$ty] { + &self.0[index] + } + } + + impl Index> for $thing { + type Output = [$ty]; + + #[inline] + fn index(&self, index: RangeFrom) -> &[$ty] { + &self.0[index] + } + } + + impl Index for $thing { + type Output = [$ty]; + + #[inline] + fn index(&self, _: RangeFull) -> &[$ty] { + &self.0[..] + } + } + } +} + +macro_rules! construct_uint { + ($name:ident, $n_words:expr) => ( + /// Little-endian large integer type + pub struct $name(pub [u64; $n_words]); + impl_array_newtype!($name, u64, $n_words); + + impl $name { + /// Conversion to u32 + #[inline] + fn low_u32(&self) -> u32 { + let &$name(ref arr) = self; + arr[0] as u32 + } + + /// Return the least number of bits needed to represent the number + #[inline] + pub fn bits(&self) -> usize { + let &$name(ref arr) = self; + for i in 1..$n_words { + if arr[$n_words - i] > 0 { return (0x40 * ($n_words - i + 1)) - arr[$n_words - i].leading_zeros() as usize; } + } + 0x40 - arr[0].leading_zeros() as usize + } + + #[inline] + pub fn bit(&self, index: usize) -> bool { + let &$name(ref arr) = self; + arr[index / 64] & (1 << (index % 64)) != 0 + } + + /// Multiplication by u32 + fn mul_u32(self, other: u32) -> $name { + let $name(ref arr) = self; + let mut carry = [0u64; $n_words]; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + let upper = other as u64 * (arr[i] >> 32); + let lower = other as u64 * (arr[i] & 0xFFFFFFFF); + if i < 3 { + carry[i + 1] += upper >> 32; + } + ret[i] = lower + (upper << 32); + } + $name(ret) + $name(carry) + } + } + + impl From for $name { + fn from(value: u64) -> $name { + let mut ret = [0; $n_words]; + ret[0] = value; + $name(ret) + } + } + + impl_map_from!($name, u8, u64); + impl_map_from!($name, u16, u64); + impl_map_from!($name, u32, u64); + + impl<'a> From<&'a [u8]> for $name { + fn from(bytes: &[u8]) -> $name { + assert!($n_words * 8 >= bytes.len()); + + let mut ret = [0; $n_words]; + for i in 0..bytes.len() { + let rev = bytes.len() - 1 - i; + let pos = rev / 8; + ret[pos] += (bytes[i] as u64) << (rev % 8) * 8; + } + $name(ret) + } + } + + impl FromStr for $name { + type Err = FromHexError; + + fn from_str(value: &str) -> Result<$name, Self::Err> { + println!("{}", value); + let bytes: &[u8] = &try!(value.from_hex()); + Ok(From::from(bytes)) + } + } + + impl Add<$name> for $name { + type Output = $name; + + fn add(self, other: $name) -> $name { + let $name(ref me) = self; + let $name(ref you) = other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + if i < $n_words - 1 && ret[i] < me[i] { + carry[i + 1] = 1; + b_carry = true; + } + } + if b_carry { $name(ret) + $name(carry) } else { $name(ret) } + } + } + + impl Sub<$name> for $name { + type Output = $name; + + #[inline] + fn sub(self, other: $name) -> $name { + self + !other + From::from(1u64) + } + } + + impl Mul<$name> for $name { + type Output = $name; + + fn mul(self, other: $name) -> $name { + let mut me = self; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); + } + me + } + } + + impl Div<$name> for $name { + type Output = $name; + + fn div(self, other: $name) -> $name { + let mut sub_copy = self; + let mut shift_copy = other; + let mut ret = [0u64; $n_words]; + + let my_bits = self.bits(); + let your_bits = other.bits(); + + // Check for division by 0 + assert!(your_bits != 0); + + // Early return in case we are dividing by a larger number than us + if my_bits < your_bits { + return $name(ret); + } + + // Bitwise long division + let mut shift = my_bits - your_bits; + shift_copy = shift_copy << shift; + loop { + if sub_copy >= shift_copy { + ret[shift / 64] |= 1 << (shift % 64); + sub_copy = sub_copy - shift_copy; + } + shift_copy = shift_copy >> 1; + if shift == 0 { break; } + shift -= 1; + } + + $name(ret) + } + } + + impl BitAnd<$name> for $name { + type Output = $name; + + #[inline] + fn bitand(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] & arr2[i]; + } + $name(ret) + } + } + + impl BitXor<$name> for $name { + type Output = $name; + + #[inline] + fn bitxor(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] ^ arr2[i]; + } + $name(ret) + } + } + + impl BitOr<$name> for $name { + type Output = $name; + + #[inline] + fn bitor(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] | arr2[i]; + } + $name(ret) + } + } + + impl Not for $name { + type Output = $name; + + #[inline] + fn not(self) -> $name { + let $name(ref arr) = self; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = !arr[i]; + } + $name(ret) + } + } + + impl Shl for $name { + type Output = $name; + + fn shl(self, shift: usize) -> $name { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in 0..$n_words { + // Shift + if bit_shift < 64 && i + word_shift < $n_words { + ret[i + word_shift] += original[i] << bit_shift; + } + // Carry + if bit_shift > 0 && i + word_shift + 1 < $n_words { + ret[i + word_shift + 1] += original[i] >> (64 - bit_shift); + } + } + $name(ret) + } + } + + impl Shr for $name { + type Output = $name; + + fn shr(self, shift: usize) -> $name { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in word_shift..$n_words { + // Shift + ret[i - word_shift] += original[i] >> bit_shift; + // Carry + if bit_shift > 0 && i < $n_words - 1 { + ret[i - word_shift] += original[i + 1] << (64 - bit_shift); + } + } + $name(ret) + } + } + + impl Ord for $name { + fn cmp(&self, other: &$name) -> Ordering { + let &$name(ref me) = self; + let &$name(ref you) = other; + for i in 0..$n_words { + if me[$n_words - 1 - i] < you[$n_words - 1 - i] { return Ordering::Less; } + if me[$n_words - 1 - i] > you[$n_words - 1 - i] { return Ordering::Greater; } + } + Ordering::Equal + } + } + + impl PartialOrd for $name { + fn partial_cmp(&self, other: &$name) -> Option { + Some(self.cmp(other)) + } + } + + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let &$name(ref data) = self; + try!(write!(f, "0x")); + for ch in data.iter().rev() { + try!(write!(f, "{:02x}", ch)); + } + Ok(()) + } + } + ); +} + +construct_uint!(U256, 4); +construct_uint!(U128, 2); + +impl From for U256 { + fn from(value: U128) -> U256 { + let U128(ref arr) = value; + let mut ret = [0; 4]; + ret[0] = arr[0]; + ret[1] = arr[1]; + U256(ret) + } +} + +#[cfg(test)] +mod tests { + use uint::U256; + use std::str::FromStr; + + #[test] + pub fn uint256_from() { + let e = U256([10, 0, 0, 0]); + + // test unsigned initialization + let ua = U256::from(10u8); + let ub = U256::from(10u16); + let uc = U256::from(10u32); + let ud = U256::from(10u64); + assert_eq!(e, ua); + assert_eq!(e, ub); + assert_eq!(e, uc); + assert_eq!(e, ud); + + // test initialization from bytes + let va = U256::from(&[10u8][..]); + assert_eq!(e, va); + + // more tests for initialization from bytes + assert_eq!(U256([0x1010, 0, 0, 0]), U256::from(&[0x10u8, 0x10][..])); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from(&[0, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from(&[1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from(&[ + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x00192437100019fa, 0x243710, 0, 0]), U256::from(&[ + 0x24u8, 0x37, 0x10, + 0, 0x19, 0x24, 0x37, 0x10, 0, 0x19, 0xfa][..])); + + // test initializtion from string + let sa = U256::from_str("0a").unwrap(); + assert_eq!(e, sa); + assert_eq!(U256([0x1010, 0, 0, 0]), U256::from_str("1010").unwrap()); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); + assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from_str("0000000012f0").unwrap()); + assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from_str("0100000000000012f0").unwrap()); + assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); + } + + #[test] + pub fn uint256_bits_test() { + assert_eq!(U256::from(255u64).bits(), 8); + assert_eq!(U256::from(256u64).bits(), 9); + assert_eq!(U256::from(300u64).bits(), 9); + assert_eq!(U256::from(60000u64).bits(), 16); + assert_eq!(U256::from(70000u64).bits(), 17); + + //// Try to read the following lines out loud quickly + let mut shl = U256::from(70000u64); + shl = shl << 100; + assert_eq!(shl.bits(), 117); + shl = shl << 100; + assert_eq!(shl.bits(), 217); + shl = shl << 100; + assert_eq!(shl.bits(), 0); + + //// Bit set check + assert!(!U256::from(10u8).bit(0)); + assert!(U256::from(10u8).bit(1)); + assert!(!U256::from(10u8).bit(2)); + assert!(U256::from(10u8).bit(3)); + assert!(!U256::from(10u8).bit(4)); + } + + #[test] + pub fn uint256_comp_test() { + let small = U256([10u64, 0, 0, 0]); + let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); + let bigger = U256([0x9C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); + let biggest = U256([0x5C8C3EE70C644118u64, 0x0209E7378231E632, 0, 1]); + + assert!(small < big); + assert!(big < bigger); + assert!(bigger < biggest); + assert!(bigger <= biggest); + assert!(biggest <= biggest); + assert!(bigger >= big); + assert!(bigger >= small); + assert!(small <= small); + } + + #[test] + pub fn uint256_arithmetic_test() { + let init = U256::from(0xDEADBEEFDEADBEEFu64); + let copy = init; + + let add = init + copy; + assert_eq!(add, U256([0xBD5B7DDFBD5B7DDEu64, 1, 0, 0])); + // Bitshifts + let shl = add << 88; + assert_eq!(shl, U256([0u64, 0xDFBD5B7DDE000000, 0x1BD5B7D, 0])); + let shr = shl >> 40; + assert_eq!(shr, U256([0x7DDE000000000000u64, 0x0001BD5B7DDFBD5B, 0, 0])); + // Increment + let incr = shr + U256::from(1u64); + assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); + // Subtraction + let sub = incr - init; + assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); + // Multiplication + let mult = sub.mul_u32(300); + assert_eq!(mult, U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0])); + // Division + assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8)); + let div = mult / U256::from(300u16); + assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); + //// TODO: bit inversion + } + + #[test] + pub fn uint256_extreme_bitshift_test() { + //// Shifting a u64 by 64 bits gives an undefined value, so make sure that + //// we're doing the Right Thing here + let init = U256::from(0xDEADBEEFDEADBEEFu64); + + assert_eq!(init << 64, U256([0, 0xDEADBEEFDEADBEEF, 0, 0])); + let add = (init << 64) + init; + assert_eq!(add, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add >> 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add << 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); + assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); + } +} + From 4fcf044eddde2382ba18ceeac21076cdb37cecb9 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 15:00:17 +0100 Subject: [PATCH 034/379] rlp encoding and decoding u256 --- src/bytes.rs | 35 +++++++++++++++++++++++++++++++++++ src/rlp.rs | 32 ++++++++++++++++++++++++++++++++ src/uint.rs | 18 ++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/src/bytes.rs b/src/bytes.rs index 06735995f..52dad3cb9 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -8,6 +8,7 @@ use std::fmt; use std::error::Error as StdError; +use uint::{U128, U256}; /// TODO: optimise some conversations pub trait ToBytes { @@ -82,6 +83,26 @@ impl_map_to_bytes!(usize, u64); impl_map_to_bytes!(u16, u64); impl_map_to_bytes!(u32, u64); +macro_rules! impl_uint_to_bytes { + ($name: ident) => { + impl ToBytes for $name { + fn to_bytes(&self) -> Vec { + let mut res= vec![]; + let count = self.to_bytes_len(); + for i in 0..count { + let j = count - 1 - i; + res.push(self.byte(j)); + } + res + } + fn to_bytes_len(&self) -> usize { (self.bits() + 7) / 8 } + } + } +} + +impl_uint_to_bytes!(U256); +impl_uint_to_bytes!(U128); + #[derive(Debug, PartialEq, Eq)] pub enum FromBytesError { UnexpectedEnd @@ -101,6 +122,7 @@ pub type FromBytesResult = Result; /// implements "Sized", so the compiler can deducate the size /// of the return type +/// TODO: check size of bytes before conversation and return appropriate error pub trait FromBytes: Sized { fn from_bytes(bytes: &[u8]) -> FromBytesResult; } @@ -149,3 +171,16 @@ macro_rules! impl_map_from_bytes { impl_map_from_bytes!(usize, u64); impl_map_from_bytes!(u16, u64); impl_map_from_bytes!(u32, u64); + +macro_rules! impl_uint_from_bytes { + ($name: ident) => { + impl FromBytes for $name { + fn from_bytes(bytes: &[u8]) -> FromBytesResult<$name> { + Ok($name::from(bytes)) + } + } + } +} + +impl_uint_from_bytes!(U256); +impl_uint_from_bytes!(U128); diff --git a/src/rlp.rs b/src/rlp.rs index cd83a8c7e..03a10348c 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -522,8 +522,10 @@ impl Encoder for BasicEncoder { #[cfg(test)] mod tests { use std::{fmt, cmp}; + use std::str::FromStr; use rlp; use rlp::{Rlp, RlpStream, Decodable}; + use uint::U256; #[test] fn rlp_at() { @@ -643,6 +645,21 @@ mod tests { run_encode_tests(tests); } + #[test] + fn encode_u256() { + let tests = vec![ + ETestPair(U256::from(0u64), vec![0x80u8]), + ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(U256::from(0xffffffffu64), vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]) + ]; + run_encode_tests(tests); + } + #[test] fn encode_str() { let tests = vec![ @@ -769,6 +786,21 @@ mod tests { run_decode_tests(tests); } + #[test] + fn decode_u256() { + let tests = vec![ + DTestPair(U256::from(0u64), vec![0x80u8]), + DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(U256::from(0xffffffffu64), vec![0x84, 0xff, 0xff, 0xff, 0xff]), + DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]) + ]; + run_decode_tests(tests); + } + #[test] fn decode_str() { let tests = vec![ diff --git a/src/uint.rs b/src/uint.rs index 149ff5efb..daba98b63 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -180,6 +180,12 @@ macro_rules! construct_uint { arr[index / 64] & (1 << (index % 64)) != 0 } + #[inline] + pub fn byte(&self, index: usize) -> u8 { + let &$name(ref arr) = self; + (arr[index / 8] >> ((index % 8)) * 8) as u8 + } + /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -507,6 +513,7 @@ mod tests { #[test] pub fn uint256_bits_test() { + assert_eq!(U256::from(0u64).bits(), 0); assert_eq!(U256::from(255u64).bits(), 8); assert_eq!(U256::from(256u64).bits(), 9); assert_eq!(U256::from(300u64).bits(), 9); @@ -523,11 +530,22 @@ mod tests { assert_eq!(shl.bits(), 0); //// Bit set check + //// 01010 assert!(!U256::from(10u8).bit(0)); assert!(U256::from(10u8).bit(1)); assert!(!U256::from(10u8).bit(2)); assert!(U256::from(10u8).bit(3)); assert!(!U256::from(10u8).bit(4)); + + //// byte check + assert_eq!(U256::from(10u8).byte(0), 10); + assert_eq!(U256::from(0xffu64).byte(0), 0xff); + assert_eq!(U256::from(0xffu64).byte(1), 0); + assert_eq!(U256::from(0x01ffu64).byte(0), 0xff); + assert_eq!(U256::from(0x01ffu64).byte(1), 0x1); + assert_eq!(U256([0u64, 0xfc, 0, 0]).byte(8), 0xfc); + assert_eq!(U256([0u64, 0, 0, u64::max_value()]).byte(31), 0xff); + assert_eq!(U256([0u64, 0, 0, (u64::max_value() >> 8) + 1]).byte(31), 0x01); } #[test] From 21a5d5418bdb9b3b8cc60149a801d300bf08e2e5 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 15:43:52 +0100 Subject: [PATCH 035/379] added benchmarks and few optimisations for rlp encoding --- benches/rlp.rs | 50 ++++++++++++++++++++++++++++++++++++++++++-------- src/bytes.rs | 2 ++ src/uint.rs | 1 - 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 10397db09..8c1fa8d58 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -10,19 +10,53 @@ extern crate test; extern crate ethcore_util; use test::Bencher; -use ethcore_util::rlp; +use std::str::FromStr; use ethcore_util::rlp::{RlpStream, Rlp, Decodable}; +use ethcore_util::uint::U256; #[bench] -fn bench_stream_value(b: &mut Bencher) { +fn bench_stream_u64_value(b: &mut Bencher) { b.iter( || { //1029 let mut stream = RlpStream::new(); - stream.append(&1029u32); + stream.append(&1029u64); let _ = stream.out().unwrap(); }); } +#[bench] +fn bench_decode_u64_value(b: &mut Bencher) { + b.iter( || { + // 1029 + let data = vec![0x82, 0x04, 0x05]; + let rlp = Rlp::new(&data); + let _ = u64::decode(&rlp).unwrap(); + }); +} + +#[bench] +fn bench_stream_u256_value(b: &mut Bencher) { + b.iter( || { + //u256 + let mut stream = RlpStream::new(); + stream.append(&U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); + let _ = stream.out().unwrap(); + }); +} + +#[bench] +fn bench_decode_u256_value(b: &mut Bencher) { + b.iter( || { + // u256 + let data = vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; + let rlp = Rlp::new(&data); + let _ = U256::decode(&rlp).unwrap(); + }); +} + #[bench] fn bench_stream_nested_empty_lists(b: &mut Bencher) { b.iter( || { @@ -41,17 +75,17 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { // [ [], [[]], [ [], [[]] ] ] let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; let rlp = Rlp::new(&data); - let v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); - let v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); - let v2a: Vec = Decodable::decode(&rlp.at(2).unwrap().at(0).unwrap()).unwrap(); - let v2b: Vec> = Decodable::decode(&rlp.at(2).unwrap().at(1).unwrap()).unwrap(); + let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); + let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); + let _v2a: Vec = Decodable::decode(&rlp.at(2).unwrap().at(0).unwrap()).unwrap(); + let _v2b: Vec> = Decodable::decode(&rlp.at(2).unwrap().at(1).unwrap()).unwrap(); }); } #[bench] fn bench_stream_1000_empty_lists(b: &mut Bencher) { b.iter( || { - let mut stream = RlpStream::new(); + let mut stream = RlpStream::new_list(1000); for _ in 0..1000 { stream.append_list(0); } diff --git a/src/bytes.rs b/src/bytes.rs index 52dad3cb9..1b331cf15 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -60,6 +60,7 @@ impl ToBytes for u64 { fn to_bytes(&self) -> Vec { let mut res= vec![]; let count = self.to_bytes_len(); + res.reserve(count); for i in 0..count { let j = count - 1 - i; res.push((*self >> (j * 8)) as u8); @@ -89,6 +90,7 @@ macro_rules! impl_uint_to_bytes { fn to_bytes(&self) -> Vec { let mut res= vec![]; let count = self.to_bytes_len(); + res.reserve(count); for i in 0..count { let j = count - 1 - i; res.push(self.byte(j)); diff --git a/src/uint.rs b/src/uint.rs index daba98b63..bc333631b 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -233,7 +233,6 @@ macro_rules! construct_uint { type Err = FromHexError; fn from_str(value: &str) -> Result<$name, Self::Err> { - println!("{}", value); let bytes: &[u8] = &try!(value.from_hex()); Ok(From::from(bytes)) } From d7dc1e5f6f792a1d5692a01ecaa8279d3bbb0160 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 16:15:57 +0100 Subject: [PATCH 036/379] fixes in benchmars --- benches/rlp.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 8c1fa8d58..73af76bbe 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -19,7 +19,7 @@ fn bench_stream_u64_value(b: &mut Bencher) { b.iter( || { //1029 let mut stream = RlpStream::new(); - stream.append(&1029u64); + stream.append(&0x1023456789abcdefu64); let _ = stream.out().unwrap(); }); } @@ -28,7 +28,7 @@ fn bench_stream_u64_value(b: &mut Bencher) { fn bench_decode_u64_value(b: &mut Bencher) { b.iter( || { // 1029 - let data = vec![0x82, 0x04, 0x05]; + let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; let rlp = Rlp::new(&data); let _ = u64::decode(&rlp).unwrap(); }); @@ -77,8 +77,9 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { let rlp = Rlp::new(&data); let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); - let _v2a: Vec = Decodable::decode(&rlp.at(2).unwrap().at(0).unwrap()).unwrap(); - let _v2b: Vec> = Decodable::decode(&rlp.at(2).unwrap().at(1).unwrap()).unwrap(); + let nested_rlp = rlp.at(2).unwrap(); + let _v2a: Vec = Decodable::decode(&nested_rlp.at(0).unwrap()).unwrap(); + let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1).unwrap()).unwrap(); }); } From 716c907dc2cd5c1289a60889c28d778464cd9fa8 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 18:36:28 +0100 Subject: [PATCH 037/379] when rewriting the buffer, reserve enougth space --- src/rlp.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index 03a10348c..3e29b783b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -459,6 +459,8 @@ impl BasicEncoder { fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { // new bytes let mut res: Vec = vec![]; + // reserve a space equal at least current space + space for length + res.reserve(self.bytes.len() + 1); { let (before_slice, after_slice) = self.bytes.split_at(pos); res.extend(before_slice); From 0b9620f3ec6580150501e9e93da66f3689264643 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 26 Nov 2015 19:09:14 +0100 Subject: [PATCH 038/379] rlp uses insert_slice --- src/lib.rs | 1 + src/rlp.rs | 30 +++++++++++------------------- src/vector.rs | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 src/vector.rs diff --git a/src/lib.rs b/src/lib.rs index ab8cac701..863f5d74c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,7 @@ pub mod hash; pub mod uint; pub mod bytes; pub mod rlp; +pub mod vector; #[test] fn it_works() { diff --git a/src/rlp.rs b/src/rlp.rs index 3e29b783b..f70062457 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -48,6 +48,7 @@ use std::cell::Cell; use std::collections::LinkedList; use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; +use vector::InsertSlice; /// rlp container #[derive(Debug)] @@ -455,27 +456,18 @@ impl BasicEncoder { } /// inserts list prefix at given position - /// TODO: optimise it, so it does not copy an array + /// TODO: optimise it further? fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { - // new bytes - let mut res: Vec = vec![]; - // reserve a space equal at least current space + space for length - res.reserve(self.bytes.len() + 1); - { - let (before_slice, after_slice) = self.bytes.split_at(pos); - res.extend(before_slice); + let mut res = vec![]; + match len { + 0...55 => res.push(0xc0u8 + len as u8), + _ => { + res.push(0x7fu8 + len.to_bytes_len() as u8); + res.extend(len.to_bytes()); + } + }; - match len { - 0...55 => res.push(0xc0u8 + len as u8), - _ => { - res.push(0x7fu8 + len.to_bytes_len() as u8); - res.extend(len.to_bytes()); - } - }; - - res.extend(after_slice); - } - self.bytes = res; + self.bytes.insert_slice(pos, &res); } /// get encoded value diff --git a/src/vector.rs b/src/vector.rs new file mode 100644 index 000000000..ce891beba --- /dev/null +++ b/src/vector.rs @@ -0,0 +1,34 @@ + +use std::ptr; + +pub trait InsertSlice { + fn insert_slice(&mut self, index: usize, elements: &[T]); +} + +/// based on `insert` function implementation from standard library +impl InsertSlice for Vec { + fn insert_slice(&mut self, index: usize, elements: &[T]) { + let e_len = elements.len(); + if e_len == 0 { + return; + } + + let len = self.len(); + assert!(index <= len); + + // space for the new element + self.reserve(e_len); + + unsafe { + { + let p = self.as_mut_ptr().offset(index as isize); + let ep = elements.as_ptr().offset(0); + // shift everything by e_len, to make space + ptr::copy(p, p.offset(e_len as isize), len - index); + // write new element + ptr::copy(ep, p, e_len); + } + self.set_len(len + e_len); + } + } +} From ebf9fa21a821b556eaa978cdb3b2dbde7a721487 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 26 Nov 2015 21:34:28 +0100 Subject: [PATCH 039/379] 64-byte hashes and reorg. --- src/hash.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hash.rs b/src/hash.rs index 224bc3699..5c659184c 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -58,7 +58,7 @@ impl_hash!(Hash2048, 256); impl_hash!(Hash4096, 512); #[test] -fn it_works() { +fn hash() { let h = Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); assert_eq!(Hash64::from_str("0123456789abcdef").unwrap(), h); assert_eq!(format!("{}", h), "0123456789abcdef"); From 91b5df8d6de226fc7f266aaa90447662cda6eefd Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 26 Nov 2015 23:24:01 +0100 Subject: [PATCH 040/379] Additional hash traits and methods --- rustfmt.toml | 1 + src/hash.rs | 72 +++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 rustfmt.toml diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 000000000..218e20321 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1 @@ +hard_tabs = true diff --git a/src/hash.rs b/src/hash.rs index 5c659184c..39233ea83 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,12 +1,31 @@ -use rustc_serialize::hex::*; -use error::EthcoreError; use std::str::FromStr; use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ops::{Index, IndexMut}; +use rustc_serialize::hex::*; +use error::EthcoreError; +use rand::Rng; +use rand::os::OsRng; macro_rules! impl_hash { ($from: ident, $size: expr) => { pub struct $from (pub [u8; $size]); + impl $from { + pub fn new() -> $from { + $from([0; $size]) + } + pub fn random() -> $from { + let mut hash = $from::new(); + hash.randomize(); + hash + } + pub fn randomize(&mut self) { + let mut rng = OsRng::new().unwrap(); + rng.fill_bytes(&mut self.0); + } + } + impl FromStr for $from { type Err = EthcoreError; @@ -27,22 +46,51 @@ macro_rules! impl_hash { try!(write!(f, "{:02x}", i)); } Ok(()) - } - } + } + } impl fmt::Display for $from { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { (self as &fmt::Debug).fmt(f) - } - } + } + } + + impl Clone for $from { + fn clone(&self) -> $from { + *self + } + } + impl Copy for $from {} + impl PartialEq for $from { - fn eq(&self, other: &Self) -> bool { + fn eq(&self, other: &Self) -> bool { for i in 0..$size { - if self.0[i] != other.0[i] { - return false; - } + if self.0[i] != other.0[i] { + return false; + } } true - } + } + } + impl Eq for $from { } + + impl Hash for $from { + fn hash(&self, state: &mut H) where H: Hasher { + state.write(&self.0); + state.finish(); + } + } + + impl Index for $from { + type Output = u8; + + fn index<'a>(&'a self, index: usize) -> &'a u8 { + &self.0[index] + } + } + impl IndexMut for $from { + fn index_mut<'a>(&'a mut self, index: usize) -> &'a mut u8 { + &mut self.0[index] + } } } } @@ -66,4 +114,4 @@ fn hash() { assert!(h == h); assert!(h != Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); assert!(h != Hash64([0; 8])); -} \ No newline at end of file +} From a2fe4a4ac135a9677e46c65b90ad4001108604c7 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 26 Nov 2015 23:44:03 +0100 Subject: [PATCH 041/379] Added rand and mio --- Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 0b7032bbb..542b5d19e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,6 @@ authors = ["Ethcore "] log = "0.3" env_logger = "0.3" rustc-serialize = "0.3" -arrayvec = "0.3" \ No newline at end of file +arrayvec = "0.3" +mio = "0.*" +rand = "0.*" From 8956d4cd5f8af43a395cf0e768ad54305fdcba6f Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 26 Nov 2015 23:45:02 +0100 Subject: [PATCH 042/379] Added rand and mio --- src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 7d9212634..44ffc106f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,8 @@ extern crate rustc_serialize; +extern crate mio; +extern crate rand; +#[macro_use] +extern crate log; pub use std::str::FromStr; @@ -9,6 +13,8 @@ pub mod bytes; pub mod rlp; pub mod vector; +//pub mod network; + #[test] fn it_works() { } From 3d35f57a036af5d957efeb4cff015c98bc10c489 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 27 Nov 2015 00:30:21 +0100 Subject: [PATCH 043/379] minor updates. --- src/hash.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hash.rs b/src/hash.rs index 39233ea83..081b0df02 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -9,6 +9,7 @@ use rand::os::OsRng; macro_rules! impl_hash { ($from: ident, $size: expr) => { + #[derive(Eq)] pub struct $from (pub [u8; $size]); impl $from { @@ -71,7 +72,6 @@ macro_rules! impl_hash { true } } - impl Eq for $from { } impl Hash for $from { fn hash(&self, state: &mut H) where H: Hasher { From f4927619da82ccdc33ec504031b37128e9aedba1 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 11:30:14 +0100 Subject: [PATCH 044/379] added tiny-keccak to Cargo.toml --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 542b5d19e..4a1917e4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,4 @@ rustc-serialize = "0.3" arrayvec = "0.3" mio = "0.*" rand = "0.*" +tiny-keccak = "0.1" From 4ac86c4e30bb8f1c9577726487328d43506cadee Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 13:16:32 +0100 Subject: [PATCH 045/379] updated tiny-keccak to 0.2, added rocksdb and renamed HashX to HX --- Cargo.toml | 3 ++- src/db.rs | 0 src/hash.rs | 24 ++++++++++++------------ src/lib.rs | 3 +++ 4 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 src/db.rs diff --git a/Cargo.toml b/Cargo.toml index 4a1917e4f..cc0953f8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,5 @@ rustc-serialize = "0.3" arrayvec = "0.3" mio = "0.*" rand = "0.*" -tiny-keccak = "0.1" +tiny-keccak = "0.2" +rocksdb = "0.2.1" diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/hash.rs b/src/hash.rs index 081b0df02..17f1c6c71 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -95,23 +95,23 @@ macro_rules! impl_hash { } } -impl_hash!(Hash64, 8); -impl_hash!(Hash128, 16); +impl_hash!(H64, 8); +impl_hash!(H128, 16); impl_hash!(Address, 20); -impl_hash!(Hash256, 32); -impl_hash!(Hash512, 64); -impl_hash!(Hash520, 65); -impl_hash!(Hash1024, 128); -impl_hash!(Hash2048, 256); -impl_hash!(Hash4096, 512); +impl_hash!(H256, 32); +impl_hash!(H512, 64); +impl_hash!(H520, 65); +impl_hash!(H1024, 128); +impl_hash!(H2048, 256); +impl_hash!(H4096, 512); #[test] fn hash() { - let h = Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); - assert_eq!(Hash64::from_str("0123456789abcdef").unwrap(), h); + let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); + assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); assert_eq!(format!("{}", h), "0123456789abcdef"); assert_eq!(format!("{:?}", h), "0123456789abcdef"); assert!(h == h); - assert!(h != Hash64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); - assert!(h != Hash64([0; 8])); + assert!(h != H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); + assert!(h != H64([0; 8])); } diff --git a/src/lib.rs b/src/lib.rs index 44ffc106f..9d1d25808 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,8 @@ extern crate rustc_serialize; extern crate mio; extern crate rand; +extern crate rocksdb; +extern crate tiny_keccak; #[macro_use] extern crate log; @@ -12,6 +14,7 @@ pub mod uint; pub mod bytes; pub mod rlp; pub mod vector; +pub mod db; //pub mod network; From 3def0b9609ccf511895fc22149bb49c978b6c740 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 27 Nov 2015 13:17:28 +0100 Subject: [PATCH 046/379] Add uint constants. --- src/lib.rs | 2 ++ src/uint.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 44ffc106f..e15f33c56 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,8 @@ pub mod vector; //pub mod network; +pub type Bytes = Vec; + #[test] fn it_works() { } diff --git a/src/uint.rs b/src/uint.rs index bc333631b..c8e9d84de 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -461,6 +461,10 @@ impl From for U256 { } } +pub const ZERO_U256: U256 = U256([0x00u64; 4]); +pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); +pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); + #[cfg(test)] mod tests { use uint::U256; From 006fe20287978c07a5908fa0f182e9fa7a2f325d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 27 Nov 2015 17:54:33 +0100 Subject: [PATCH 047/379] Add SHA3 --- Cargo.toml | 2 +- src/bytes.rs | 229 +++++++++++++++++++++++++++------------------------ src/hash.rs | 11 +++ src/lib.rs | 1 + src/sha3.rs | 29 +++++++ 5 files changed, 165 insertions(+), 107 deletions(-) create mode 100644 src/sha3.rs diff --git a/Cargo.toml b/Cargo.toml index cc0953f8f..19c64113c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,5 +13,5 @@ rustc-serialize = "0.3" arrayvec = "0.3" mio = "0.*" rand = "0.*" -tiny-keccak = "0.2" +tiny-keccak = "0.3" rocksdb = "0.2.1" diff --git a/src/bytes.rs b/src/bytes.rs index 1b331cf15..73592643d 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -10,74 +10,91 @@ use std::fmt; use std::error::Error as StdError; use uint::{U128, U256}; +pub trait BytesConvertable { + fn bytes(&self) -> &[u8]; +} + +impl<'a> BytesConvertable for &'a [u8] { + fn bytes(&self) -> &[u8] { self } +} + +impl BytesConvertable for Vec { + fn bytes(&self) -> &[u8] { self } +} + +#[test] +fn bytes_convertable() { + assert_eq!(vec![0x12u8, 0x34].bytes(), &[0x12u8, 0x34]); +} + /// TODO: optimise some conversations pub trait ToBytes { - fn to_bytes(&self) -> Vec; - fn to_bytes_len(&self) -> usize { self.to_bytes().len() } - fn first_byte(&self) -> Option { self.to_bytes().first().map(|&x| { x })} + fn to_bytes(&self) -> Vec; + fn to_bytes_len(&self) -> usize { self.to_bytes().len() } + fn first_byte(&self) -> Option { self.to_bytes().first().map(|&x| { x })} } impl <'a> ToBytes for &'a str { - fn to_bytes(&self) -> Vec { - From::from(*self) - } - - fn to_bytes_len(&self) -> usize { self.len() } + fn to_bytes(&self) -> Vec { + From::from(*self) + } + + fn to_bytes_len(&self) -> usize { self.len() } } impl ToBytes for String { - fn to_bytes(&self) -> Vec { - let s: &str = self.as_ref(); - From::from(s) - } - - fn to_bytes_len(&self) -> usize { self.len() } + fn to_bytes(&self) -> Vec { + let s: &str = self.as_ref(); + From::from(s) + } + + fn to_bytes_len(&self) -> usize { self.len() } } impl ToBytes for u8 { - fn to_bytes(&self) -> Vec { - match *self { - 0 => vec![], - _ => vec![*self] - } - } + fn to_bytes(&self) -> Vec { + match *self { + 0 => vec![], + _ => vec![*self] + } + } - fn to_bytes_len(&self) -> usize { - match *self { - 0 => 0, - _ => 1 - } - } - fn first_byte(&self) -> Option { - match *self { - 0 => None, - _ => Some(*self) - } - } + fn to_bytes_len(&self) -> usize { + match *self { + 0 => 0, + _ => 1 + } + } + fn first_byte(&self) -> Option { + match *self { + 0 => None, + _ => Some(*self) + } + } } impl ToBytes for u64 { - fn to_bytes(&self) -> Vec { - let mut res= vec![]; - let count = self.to_bytes_len(); - res.reserve(count); - for i in 0..count { - let j = count - 1 - i; - res.push((*self >> (j * 8)) as u8); - } - res - } + fn to_bytes(&self) -> Vec { + let mut res= vec![]; + let count = self.to_bytes_len(); + res.reserve(count); + for i in 0..count { + let j = count - 1 - i; + res.push((*self >> (j * 8)) as u8); + } + res + } - fn to_bytes_len(&self) -> usize { 8 - self.leading_zeros() as usize / 8 } + fn to_bytes_len(&self) -> usize { 8 - self.leading_zeros() as usize / 8 } } macro_rules! impl_map_to_bytes { - ($from: ident, $to: ty) => { - impl ToBytes for $from { - fn to_bytes(&self) -> Vec { (*self as $to).to_bytes() } - fn to_bytes_len(&self) -> usize { (*self as $to).to_bytes_len() } - } - } + ($from: ident, $to: ty) => { + impl ToBytes for $from { + fn to_bytes(&self) -> Vec { (*self as $to).to_bytes() } + fn to_bytes_len(&self) -> usize { (*self as $to).to_bytes_len() } + } + } } impl_map_to_bytes!(usize, u64); @@ -85,21 +102,21 @@ impl_map_to_bytes!(u16, u64); impl_map_to_bytes!(u32, u64); macro_rules! impl_uint_to_bytes { - ($name: ident) => { - impl ToBytes for $name { - fn to_bytes(&self) -> Vec { - let mut res= vec![]; - let count = self.to_bytes_len(); - res.reserve(count); - for i in 0..count { - let j = count - 1 - i; - res.push(self.byte(j)); - } - res - } - fn to_bytes_len(&self) -> usize { (self.bits() + 7) / 8 } - } - } + ($name: ident) => { + impl ToBytes for $name { + fn to_bytes(&self) -> Vec { + let mut res= vec![]; + let count = self.to_bytes_len(); + res.reserve(count); + for i in 0..count { + let j = count - 1 - i; + res.push(self.byte(j)); + } + res + } + fn to_bytes_len(&self) -> usize { (self.bits() + 7) / 8 } + } + } } impl_uint_to_bytes!(U256); @@ -107,17 +124,17 @@ impl_uint_to_bytes!(U128); #[derive(Debug, PartialEq, Eq)] pub enum FromBytesError { - UnexpectedEnd + UnexpectedEnd } impl StdError for FromBytesError { - fn description(&self) -> &str { "from_bytes error" } + fn description(&self) -> &str { "from_bytes error" } } impl fmt::Display for FromBytesError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } } pub type FromBytesResult = Result; @@ -126,48 +143,48 @@ pub type FromBytesResult = Result; /// of the return type /// TODO: check size of bytes before conversation and return appropriate error pub trait FromBytes: Sized { - fn from_bytes(bytes: &[u8]) -> FromBytesResult; + fn from_bytes(bytes: &[u8]) -> FromBytesResult; } impl FromBytes for String { - fn from_bytes(bytes: &[u8]) -> FromBytesResult { - Ok(::std::str::from_utf8(bytes).unwrap().to_string()) - } + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + Ok(::std::str::from_utf8(bytes).unwrap().to_string()) + } } impl FromBytes for u8 { - fn from_bytes(bytes: &[u8]) -> FromBytesResult { - match bytes.len() { - 0 => Ok(0), - _ => Ok(bytes[0]) - } - } + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + match bytes.len() { + 0 => Ok(0), + _ => Ok(bytes[0]) + } + } } impl FromBytes for u64 { - fn from_bytes(bytes: &[u8]) -> FromBytesResult { - match bytes.len() { - 0 => Ok(0), - l => { - let mut res = 0u64; - for i in 0..l { - let shift = (l - 1 - i) * 8; - res = res + ((bytes[i] as u64) << shift); - } - Ok(res) - } - } - } + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + match bytes.len() { + 0 => Ok(0), + l => { + let mut res = 0u64; + for i in 0..l { + let shift = (l - 1 - i) * 8; + res = res + ((bytes[i] as u64) << shift); + } + Ok(res) + } + } + } } macro_rules! impl_map_from_bytes { - ($from: ident, $to: ident) => { - impl FromBytes for $from { - fn from_bytes(bytes: &[u8]) -> FromBytesResult<$from> { - $to::from_bytes(bytes).map(| x | { x as $from }) - } - } - } + ($from: ident, $to: ident) => { + impl FromBytes for $from { + fn from_bytes(bytes: &[u8]) -> FromBytesResult<$from> { + $to::from_bytes(bytes).map(| x | { x as $from }) + } + } + } } impl_map_from_bytes!(usize, u64); @@ -175,13 +192,13 @@ impl_map_from_bytes!(u16, u64); impl_map_from_bytes!(u32, u64); macro_rules! impl_uint_from_bytes { - ($name: ident) => { - impl FromBytes for $name { - fn from_bytes(bytes: &[u8]) -> FromBytesResult<$name> { - Ok($name::from(bytes)) - } - } - } + ($name: ident) => { + impl FromBytes for $name { + fn from_bytes(bytes: &[u8]) -> FromBytesResult<$name> { + Ok($name::from(bytes)) + } + } + } } impl_uint_from_bytes!(U256); diff --git a/src/hash.rs b/src/hash.rs index 17f1c6c71..044153e0c 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -6,6 +6,7 @@ use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; use rand::os::OsRng; +use bytes::BytesConvertable; macro_rules! impl_hash { ($from: ident, $size: expr) => { @@ -25,6 +26,16 @@ macro_rules! impl_hash { let mut rng = OsRng::new().unwrap(); rng.fill_bytes(&mut self.0); } + + pub fn mut_bytes(&mut self) -> &mut [u8; $size] { + &mut self.0 + } + } + + impl BytesConvertable for $from { + fn bytes(&self) -> &[u8] { + &self.0 + } } impl FromStr for $from { diff --git a/src/lib.rs b/src/lib.rs index 5b6676169..7c480c02b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod bytes; pub mod rlp; pub mod vector; pub mod db; +pub mod sha3; //pub mod network; diff --git a/src/sha3.rs b/src/sha3.rs new file mode 100644 index 000000000..61c5c26f8 --- /dev/null +++ b/src/sha3.rs @@ -0,0 +1,29 @@ +use std::mem::uninitialized; +use tiny_keccak::keccak_256; +use bytes::BytesConvertable; +use hash::H256; + +trait Hashable { + fn sha3(&self) -> H256; +} + +impl Hashable for T where T: BytesConvertable { + fn sha3(&self) -> H256 { + unsafe { + let mut ret: H256 = uninitialized(); + keccak_256(self.bytes(), ret.mut_bytes()); + ret + } + } +} + +#[test] +fn sha3_empty() { + use std::str::FromStr; + assert_eq!((&[0u8; 0][..]).sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); +} +#[test] +fn sha3_as() { + use std::str::FromStr; + assert_eq!((&[0x41u8; 32][..]).sha3(), H256::from_str("59cad5948673622c1d64e2322488bf01619f7ff45789741b15a9f782ce9290a8").unwrap()); +} From 6477b2ad69fde88002921dbf51128c53590ced78 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 18:15:44 +0100 Subject: [PATCH 048/379] implements "bytes" for all fixed size arrays --- src/bytes.rs | 17 +++++++++++++++++ src/sha3.rs | 5 +++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 73592643d..870c9eef3 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -22,9 +22,26 @@ impl BytesConvertable for Vec { fn bytes(&self) -> &[u8] { self } } +macro_rules! impl_bytes_convertable_for_array { + ($zero: expr) => (); + ($len: expr, $($idx: expr),*) => { + impl BytesConvertable for [u8; $len] { + fn bytes(&self) -> &[u8] { self } + } + impl_bytes_convertable_for_array! { $($idx),* } + } +} + +// two -1 at the end is not expanded +impl_bytes_convertable_for_array! { + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -1 +} + #[test] fn bytes_convertable() { assert_eq!(vec![0x12u8, 0x34].bytes(), &[0x12u8, 0x34]); + assert_eq!([0u8; 0].bytes(), &[]); } /// TODO: optimise some conversations diff --git a/src/sha3.rs b/src/sha3.rs index 61c5c26f8..e2866d288 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -20,10 +20,11 @@ impl Hashable for T where T: BytesConvertable { #[test] fn sha3_empty() { use std::str::FromStr; - assert_eq!((&[0u8; 0][..]).sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); + assert_eq!([0u8; 0].sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); } #[test] fn sha3_as() { use std::str::FromStr; - assert_eq!((&[0x41u8; 32][..]).sha3(), H256::from_str("59cad5948673622c1d64e2322488bf01619f7ff45789741b15a9f782ce9290a8").unwrap()); + assert_eq!([0x41u8; 32].sha3(), H256::from_str("59cad5948673622c1d64e2322488bf01619f7ff45789741b15a9f782ce9290a8").unwrap()); } + From b59652837768c999462880bfbb77661a31debe90 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 18:18:49 +0100 Subject: [PATCH 049/379] fixed typo in comment --- src/bytes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bytes.rs b/src/bytes.rs index 870c9eef3..1b6ce67d7 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -32,7 +32,7 @@ macro_rules! impl_bytes_convertable_for_array { } } -// two -1 at the end is not expanded +// -1 at the end is not expanded impl_bytes_convertable_for_array! { 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -1 From 1eacff7e3d3074462f0781c926ac445cd1e37f7d Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 20:10:31 +0100 Subject: [PATCH 050/379] FixedHash trait --- src/bloom.rs | 7 +++++++ src/hash.rs | 42 +++++++++++++++++++++++++++--------------- src/lib.rs | 1 + src/sha3.rs | 2 +- 4 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 src/bloom.rs diff --git a/src/bloom.rs b/src/bloom.rs new file mode 100644 index 000000000..fe619fae6 --- /dev/null +++ b/src/bloom.rs @@ -0,0 +1,7 @@ +use bytes::BytesConvertable; +// use hash::FixedHash; + +pub trait Bloomable { + fn shift_bloom(&mut self, bytes: &T) where T: BytesConvertable; + fn contains_bloom(&self, bytes: &T) -> bool where T: BytesConvertable; +} diff --git a/src/hash.rs b/src/hash.rs index 044153e0c..fc89b4ab9 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -8,27 +8,22 @@ use rand::Rng; use rand::os::OsRng; use bytes::BytesConvertable; +/// types implementing FixedHash must be also BytesConvertable +pub trait FixedHash: BytesConvertable { + fn random() -> Self; + fn randomize(&mut self); + fn mut_bytes(&mut self) -> &mut [u8]; +} + macro_rules! impl_hash { ($from: ident, $size: expr) => { #[derive(Eq)] pub struct $from (pub [u8; $size]); - impl $from { - pub fn new() -> $from { - $from([0; $size]) - } - pub fn random() -> $from { - let mut hash = $from::new(); - hash.randomize(); - hash - } - pub fn randomize(&mut self) { - let mut rng = OsRng::new().unwrap(); - rng.fill_bytes(&mut self.0); - } - pub fn mut_bytes(&mut self) -> &mut [u8; $size] { - &mut self.0 + impl $from { + fn new() -> $from { + $from([0; $size]) } } @@ -38,6 +33,23 @@ macro_rules! impl_hash { } } + impl FixedHash for $from { + fn random() -> $from { + let mut hash = $from::new(); + hash.randomize(); + hash + } + + fn randomize(&mut self) { + let mut rng = OsRng::new().unwrap(); + rng.fill_bytes(&mut self.0); + } + + fn mut_bytes(&mut self) -> &mut [u8] { + &mut self.0 + } + } + impl FromStr for $from { type Err = EthcoreError; diff --git a/src/lib.rs b/src/lib.rs index 7c480c02b..a52e35925 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod rlp; pub mod vector; pub mod db; pub mod sha3; +pub mod bloom; //pub mod network; diff --git a/src/sha3.rs b/src/sha3.rs index e2866d288..b7eec2a5f 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,7 +1,7 @@ use std::mem::uninitialized; use tiny_keccak::keccak_256; use bytes::BytesConvertable; -use hash::H256; +use hash::{FixedHash, H256}; trait Hashable { fn sha3(&self) -> H256; From f8bd7da111e07d31e5b8037503479be0e7b09198 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 20:59:28 +0100 Subject: [PATCH 051/379] formatted rlp.rs --- src/rlp.rs | 1176 +++++++++++++++++++++++++++------------------------- 1 file changed, 607 insertions(+), 569 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index f70062457..ee74c3a33 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -2,13 +2,13 @@ //! //! Types implementing `Endocable` and `Decodable` traits //! can be easily coverted to and from rlp -//! +//! //! # Examples: -//! +//! //! ```rust //! extern crate ethcore_util; //! use ethcore_util::rlp::{RlpStream}; -//! +//! //! fn encode_value() { //! // 1029 //! let mut stream = RlpStream::new(); @@ -24,7 +24,7 @@ //! let out = stream.out().unwrap(); //! assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); //! } -//! +//! //! fn encode_list2() { //! // [ [], [[]], [ [], [[]] ] ] //! let mut stream = RlpStream::new_list(3); @@ -52,552 +52,591 @@ use vector::InsertSlice; /// rlp container #[derive(Debug)] -pub struct Rlp<'a>{ - bytes: &'a [u8], - cache: Cell +pub struct Rlp<'a> { + bytes: &'a [u8], + cache: Cell, } /// rlp offset #[derive(Copy, Clone, Debug)] struct OffsetCache { - index: usize, - offset: usize + index: usize, + offset: usize, } impl OffsetCache { - fn new(index: usize, offset: usize) -> OffsetCache { - OffsetCache { index: index, offset: offset } - } + fn new(index: usize, offset: usize) -> OffsetCache { + OffsetCache { + index: index, + offset: offset, + } + } } /// stores basic information about item struct ItemInfo { - prefix_len: usize, - value_len: usize + prefix_len: usize, + value_len: usize, } impl ItemInfo { - fn new(prefix_len: usize, value_len: usize) -> ItemInfo { - ItemInfo { prefix_len: prefix_len, value_len: value_len } - } + fn new(prefix_len: usize, value_len: usize) -> ItemInfo { + ItemInfo { + prefix_len: prefix_len, + value_len: value_len, + } + } } #[derive(Debug, PartialEq, Eq)] pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeValue, - BadRlp, + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeValue, + BadRlp, } impl StdError for DecoderError { - fn description(&self) -> &str { "builder error" } + fn description(&self) -> &str { + "builder error" + } } impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } } impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { DecoderError::FromBytesError(err) } + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } } -impl <'a>Rlp<'a> { - /// returns new instance of `Rlp` - pub fn new(bytes: &'a[u8]) -> Rlp<'a> { - Rlp { - bytes: bytes, - cache: Cell::new(OffsetCache::new(usize::max_value(), 0)) - } - } +impl<'a> Rlp<'a> { + /// returns new instance of `Rlp` + pub fn new(bytes: &'a [u8]) -> Rlp<'a> { + Rlp { + bytes: bytes, + cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), + } + } - /// get container subset at given index - /// - /// paren container caches searched position - pub fn at(&self, index: usize) -> Result, DecoderError> { - if !self.is_list() { - return Err(DecoderError::RlpExpectedToBeList); - } + /// get container subset at given index + /// + /// paren container caches searched position + pub fn at(&self, index: usize) -> Result, DecoderError> { + if !self.is_list() { + return Err(DecoderError::RlpExpectedToBeList); + } - // move to cached position if it's index is less or equal to - // current search index, otherwise move to beginning of list - let c = self.cache.get(); - let (mut bytes, to_skip) = match c.index <= index { - true => (try!(Rlp::consume(self.bytes, c.offset)), index - c.index), - false => (try!(self.consume_list_prefix()), index) - }; + // move to cached position if it's index is less or equal to + // current search index, otherwise move to beginning of list + let c = self.cache.get(); + let (mut bytes, to_skip) = match c.index <= index { + true => (try!(Rlp::consume(self.bytes, c.offset)), index - c.index), + false => (try!(self.consume_list_prefix()), index), + }; - // skip up to x items - bytes = try!(Rlp::consume_items(bytes, to_skip)); + // skip up to x items + bytes = try!(Rlp::consume_items(bytes, to_skip)); - // update the cache - self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); + // update the cache + self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); - // construct new rlp - let found = try!(Rlp::item_info(bytes)); - Ok(Rlp::new(&bytes[0..found.prefix_len + found.value_len])) - } + // construct new rlp + let found = try!(Rlp::item_info(bytes)); + Ok(Rlp::new(&bytes[0..found.prefix_len + found.value_len])) + } - /// returns true if rlp is a list - pub fn is_list(&self) -> bool { - self.bytes.len() > 0 && self.bytes[0] >= 0xc0 - } + /// returns true if rlp is a list + pub fn is_list(&self) -> bool { + self.bytes.len() > 0 && self.bytes[0] >= 0xc0 + } - /// returns true if rlp is a value - pub fn is_value(&self) -> bool { - self.bytes.len() > 0 && self.bytes[0] <= 0xbf - } + /// returns true if rlp is a value + pub fn is_value(&self) -> bool { + self.bytes.len() > 0 && self.bytes[0] <= 0xbf + } - /// returns rlp iterator - pub fn iter(&'a self) -> RlpIterator<'a> { - self.into_iter() - } + /// returns rlp iterator + pub fn iter(&'a self) -> RlpIterator<'a> { + self.into_iter() + } - /// consumes first found prefix - fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { - let item = try!(Rlp::item_info(self.bytes)); - let bytes = try!(Rlp::consume(self.bytes, item.prefix_len)); - Ok(bytes) - } + /// consumes first found prefix + fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { + let item = try!(Rlp::item_info(self.bytes)); + let bytes = try!(Rlp::consume(self.bytes, item.prefix_len)); + Ok(bytes) + } - /// consumes fixed number of items - fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { - let mut result = bytes; - for _ in 0..items { - let i = try!(Rlp::item_info(result)); - result = try!(Rlp::consume(result, (i.prefix_len + i.value_len))); - } - Ok(result) - } + /// consumes fixed number of items + fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { + let mut result = bytes; + for _ in 0..items { + let i = try!(Rlp::item_info(result)); + result = try!(Rlp::consume(result, (i.prefix_len + i.value_len))); + } + Ok(result) + } - /// return first item info - fn item_info(bytes: &[u8]) -> Result { - let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => ItemInfo::new(0, 1), - Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let prefix_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); - ItemInfo::new(prefix_len, value_len) - } - Some(l @ 0xc0...0xf7) => ItemInfo::new(1, l as usize - 0xc0), - Some(l @ 0xf8...0xff) => { - let len_of_len = l as usize - 0xf7; - let prefix_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); - ItemInfo::new(prefix_len, value_len) - }, - _ => return Err(DecoderError::BadRlp) - }; + /// return first item info + fn item_info(bytes: &[u8]) -> Result { + let item = match bytes.first().map(|&x| x) { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => ItemInfo::new(0, 1), + Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let prefix_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); + ItemInfo::new(prefix_len, value_len) + } + Some(l @ 0xc0...0xf7) => ItemInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let prefix_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); + ItemInfo::new(prefix_len, value_len) + } + _ => return Err(DecoderError::BadRlp), + }; - match item.prefix_len + item.value_len <= bytes.len() { - true => Ok(item), - false => Err(DecoderError::RlpIsTooShort) - } - } + match item.prefix_len + item.value_len <= bytes.len() { + true => Ok(item), + false => Err(DecoderError::RlpIsTooShort), + } + } - /// consumes slice prefix of length `len` - fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { - match bytes.len() >= len { - true => Ok(&bytes[len..]), - false => Err(DecoderError::RlpIsTooShort) - } - } + /// consumes slice prefix of length `len` + fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { + match bytes.len() >= len { + true => Ok(&bytes[len..]), + false => Err(DecoderError::RlpIsTooShort), + } + } } /// non-consuming rlp iterator pub struct RlpIterator<'a> { - rlp: &'a Rlp<'a>, - index: usize + rlp: &'a Rlp<'a>, + index: usize, } -impl <'a> IntoIterator for &'a Rlp<'a> { - type Item = Rlp<'a>; +impl<'a> IntoIterator for &'a Rlp<'a> { + type Item = Rlp<'a>; type IntoIter = RlpIterator<'a>; - fn into_iter(self) -> Self::IntoIter { - RlpIterator { rlp: self, index: 0 } - } + fn into_iter(self) -> Self::IntoIter { + RlpIterator { + rlp: self, + index: 0, + } + } } -impl <'a> Iterator for RlpIterator<'a> { - type Item = Rlp<'a>; +impl<'a> Iterator for RlpIterator<'a> { + type Item = Rlp<'a>; - fn next(&mut self) -> Option> { - let index = self.index; - let result = self.rlp.at(index).ok(); - self.index += 1; - result - } + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.at(index).ok(); + self.index += 1; + result + } } /// shortcut function to decode a Rlp `&[u8]` into an object -pub fn decode(bytes: &[u8]) -> Result where T: Decodable { - let rlp = Rlp::new(bytes); - T::decode(&rlp) +pub fn decode(bytes: &[u8]) -> Result + where T: Decodable +{ + let rlp = Rlp::new(bytes); + T::decode(&rlp) } pub trait Decodable: Sized { - fn decode(rlp: &Rlp) -> Result; + fn decode(rlp: &Rlp) -> Result; } -impl Decodable for T where T: FromBytes { - fn decode(rlp: &Rlp) -> Result { - match rlp.is_value() { - true => BasicDecoder::read_value(rlp.bytes), - false => Err(DecoderError::RlpExpectedToBeValue) - } - } +impl Decodable for T where T: FromBytes +{ + fn decode(rlp: &Rlp) -> Result { + match rlp.is_value() { + true => BasicDecoder::read_value(rlp.bytes), + false => Err(DecoderError::RlpExpectedToBeValue), + } + } } -impl Decodable for Vec where T: Decodable { - fn decode(rlp: &Rlp) -> Result { - match rlp.is_list() { - true => rlp.iter().map(|rlp| T::decode(&rlp)).collect(), - false => Err(DecoderError::RlpExpectedToBeList) - } - } +impl Decodable for Vec where T: Decodable +{ + fn decode(rlp: &Rlp) -> Result { + match rlp.is_list() { + true => rlp.iter().map(|rlp| T::decode(&rlp)).collect(), + false => Err(DecoderError::RlpExpectedToBeList), + } + } } pub trait Decoder { - fn read_value(bytes: &[u8]) -> Result where T: FromBytes; + fn read_value(bytes: &[u8]) -> Result where T: FromBytes; } struct BasicDecoder; impl Decoder for BasicDecoder { - fn read_value(bytes: &[u8]) -> Result where T: FromBytes { - match bytes.first().map(|&x| x) { - // rlp is too short - None => Err(DecoderError::RlpIsTooShort), - // single byt value - Some(l @ 0...0x7f) => Ok(try!(T::from_bytes(&[l]))), - // 0-55 bytes - Some(l @ 0x80...0xb7) => Ok(try!(T::from_bytes(&bytes[1..(1 + l as usize - 0x80)]))), - // longer than 55 bytes - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let begin_of_value = 1 as usize + len_of_len; - let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); - Ok(try!(T::from_bytes(&bytes[begin_of_value..begin_of_value + len]))) - }, - _ => Err(DecoderError::BadRlp) - } - } + fn read_value(bytes: &[u8]) -> Result + where T: FromBytes + { + match bytes.first().map(|&x| x) { + // rlp is too short + None => Err(DecoderError::RlpIsTooShort), + // single byt value + Some(l @ 0...0x7f) => Ok(try!(T::from_bytes(&[l]))), + // 0-55 bytes + Some(l @ 0x80...0xb7) => Ok(try!(T::from_bytes(&bytes[1..(1 + l as usize - 0x80)]))), + // longer than 55 bytes + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let begin_of_value = 1 as usize + len_of_len; + let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); + Ok(try!(T::from_bytes(&bytes[begin_of_value..begin_of_value + len]))) + } + _ => Err(DecoderError::BadRlp), + } + } } #[derive(Debug)] struct ListInfo { - position: usize, - current: usize, - max: usize + position: usize, + current: usize, + max: usize, } impl ListInfo { - fn new(position: usize, max: usize) -> ListInfo { - ListInfo { - position: position, - current: 0, - max: max - } - } + fn new(position: usize, max: usize) -> ListInfo { + ListInfo { + position: position, + current: 0, + max: max, + } + } } /// container that should be used to encode rlp pub struct RlpStream { - unfinished_lists: LinkedList, - encoder: BasicEncoder + unfinished_lists: LinkedList, + encoder: BasicEncoder, } impl RlpStream { - /// create new container for values appended one after another, - /// but not being part of the same list - pub fn new() -> RlpStream { - RlpStream { - unfinished_lists: LinkedList::new(), - encoder: BasicEncoder::new() - } - } + /// create new container for values appended one after another, + /// but not being part of the same list + pub fn new() -> RlpStream { + RlpStream { + unfinished_lists: LinkedList::new(), + encoder: BasicEncoder::new(), + } + } - /// create new container for list of size `max_len` - pub fn new_list(len: usize) -> RlpStream { - let mut stream = RlpStream::new(); - stream.append_list(len); - stream - } + /// create new container for list of size `max_len` + pub fn new_list(len: usize) -> RlpStream { + let mut stream = RlpStream::new(); + stream.append_list(len); + stream + } - /// apends value to the end of stream, chainable - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { - // encode given value and add it at the end of the stream - object.encode(&mut self.encoder); + /// apends value to the end of stream, chainable + pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream + where E: Encodable + { + // encode given value and add it at the end of the stream + object.encode(&mut self.encoder); - // if list is finished, prepend the length - self.try_to_finish(); + // if list is finished, prepend the length + self.try_to_finish(); - // return chainable self - self - } + // return chainable self + self + } - /// declare appending the list of given size - pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { - // push new list - let position = self.encoder.bytes.len(); - match len { - 0 => { - // we may finish, if the appended list len is equal 0 - self.encoder.bytes.push(0xc0u8); - self.try_to_finish(); - }, - _ => self.unfinished_lists.push_back(ListInfo::new(position, len)) - } + /// declare appending the list of given size + pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { + // push new list + let position = self.encoder.bytes.len(); + match len { + 0 => { + // we may finish, if the appended list len is equal 0 + self.encoder.bytes.push(0xc0u8); + self.try_to_finish(); + } + _ => self.unfinished_lists.push_back(ListInfo::new(position, len)), + } - // return chainable self - self - } + // return chainable self + self + } - /// return true if stream is ready - pub fn is_finished(&self) -> bool { - self.unfinished_lists.back().is_none() - } + /// return true if stream is ready + pub fn is_finished(&self) -> bool { + self.unfinished_lists.back().is_none() + } - /// streams out encoded bytes - pub fn out(self) -> Result, EncoderError> { - match self.is_finished() { - true => Ok(self.encoder.out()), - false => Err(EncoderError::StreamIsUnfinished) - } - } + /// streams out encoded bytes + pub fn out(self) -> Result, EncoderError> { + match self.is_finished() { + true => Ok(self.encoder.out()), + false => Err(EncoderError::StreamIsUnfinished), + } + } - /// try to finish lists - fn try_to_finish(&mut self) -> () { - let should_finish = match self.unfinished_lists.back_mut() { - None => false, - Some(ref mut x) => { - x.current += 1; - x.current == x.max - } - }; + /// try to finish lists + fn try_to_finish(&mut self) -> () { + let should_finish = match self.unfinished_lists.back_mut() { + None => false, + Some(ref mut x) => { + x.current += 1; + x.current == x.max + } + }; - if should_finish { - let x = self.unfinished_lists.pop_back().unwrap(); - let len = self.encoder.bytes.len() - x.position; - self.encoder.insert_list_len_at_pos(len, x.position); - self.try_to_finish(); - } - } + if should_finish { + let x = self.unfinished_lists.pop_back().unwrap(); + let len = self.encoder.bytes.len() - x.position; + self.encoder.insert_list_len_at_pos(len, x.position); + self.try_to_finish(); + } + } } /// shortcut function to encode a `T: Encodable` into a Rlp `Vec` -pub fn encode(object: &E) -> Vec where E: Encodable { - let mut encoder = BasicEncoder::new(); - object.encode(&mut encoder); - encoder.out() +pub fn encode(object: &E) -> Vec + where E: Encodable +{ + let mut encoder = BasicEncoder::new(); + object.encode(&mut encoder); + encoder.out() } #[derive(Debug)] pub enum EncoderError { - StreamIsUnfinished + StreamIsUnfinished, } impl StdError for EncoderError { - fn description(&self) -> &str { "encoder error" } + fn description(&self) -> &str { + "encoder error" + } } impl fmt::Display for EncoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } } pub trait Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder; + fn encode(&self, encoder: &mut E) -> () where E: Encoder; } pub trait Encoder { - fn emit_value(&mut self, bytes: &[u8]) -> (); - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); + fn emit_value(&mut self, bytes: &[u8]) -> (); + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); } -impl Encodable for T where T: ToBytes { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(&self.to_bytes()) - } +impl Encodable for T where T: ToBytes +{ + fn encode(&self, encoder: &mut E) -> () + where E: Encoder + { + encoder.emit_value(&self.to_bytes()) + } } -impl <'a, T> Encodable for &'a [T] where T: Encodable + 'a { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_list(|e| { - // insert all list elements - for el in self.iter() { - el.encode(e); - } - }) - } +impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a +{ + fn encode(&self, encoder: &mut E) -> () + where E: Encoder + { + encoder.emit_list(|e| { + // insert all list elements + for el in self.iter() { + el.encode(e); + } + }) + } } -impl Encodable for Vec where T: Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - let r: &[T] = self.as_ref(); - r.encode(encoder) - } +impl Encodable for Vec where T: Encodable +{ + fn encode(&self, encoder: &mut E) -> () + where E: Encoder + { + let r: &[T] = self.as_ref(); + r.encode(encoder) + } } struct BasicEncoder { - bytes: Vec + bytes: Vec, } impl BasicEncoder { - fn new() -> BasicEncoder { - BasicEncoder { bytes: vec![] } - } + fn new() -> BasicEncoder { + BasicEncoder { bytes: vec![] } + } - /// inserts list prefix at given position - /// TODO: optimise it further? - fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { - let mut res = vec![]; - match len { - 0...55 => res.push(0xc0u8 + len as u8), - _ => { - res.push(0x7fu8 + len.to_bytes_len() as u8); - res.extend(len.to_bytes()); - } - }; + /// inserts list prefix at given position + /// TODO: optimise it further? + fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { + let mut res = vec![]; + match len { + 0...55 => res.push(0xc0u8 + len as u8), + _ => { + res.push(0x7fu8 + len.to_bytes_len() as u8); + res.extend(len.to_bytes()); + } + }; - self.bytes.insert_slice(pos, &res); - } + self.bytes.insert_slice(pos, &res); + } - /// get encoded value - fn out(self) -> Vec { - self.bytes - } + /// get encoded value + fn out(self) -> Vec { + self.bytes + } } impl Encoder for BasicEncoder { - fn emit_value(&mut self, bytes: &[u8]) -> () { - match bytes.len() { - // just 0 - 0 => self.bytes.push(0x80u8), - // byte is its own encoding - 1 if bytes[0] < 0x80 => self.bytes.extend(bytes), - // (prefix + length), followed by the string - len @ 1 ... 55 => { - self.bytes.push(0x80u8 + len as u8); - self.bytes.extend(bytes); - } - // (prefix + length of length), followed by the length, followd by the string - len => { - self.bytes.push(0xb7 + len.to_bytes_len() as u8); - self.bytes.extend(len.to_bytes()); - self.bytes.extend(bytes); - } - } - } + fn emit_value(&mut self, bytes: &[u8]) -> () { + match bytes.len() { + // just 0 + 0 => self.bytes.push(0x80u8), + // byte is its own encoding + 1 if bytes[0] < 0x80 => self.bytes.extend(bytes), + // (prefix + length), followed by the string + len @ 1 ... 55 => { + self.bytes.push(0x80u8 + len as u8); + self.bytes.extend(bytes); + } + // (prefix + length of length), followed by the length, followd by the string + len => { + self.bytes.push(0xb7 + len.to_bytes_len() as u8); + self.bytes.extend(len.to_bytes()); + self.bytes.extend(bytes); + } + } + } - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () { - // get len before inserting a list - let before_len = self.bytes.len(); + fn emit_list(&mut self, f: F) -> () + where F: FnOnce(&mut Self) -> () + { + // get len before inserting a list + let before_len = self.bytes.len(); - // insert all list elements - f(self); + // insert all list elements + f(self); - // get len after inserting a list - let after_len = self.bytes.len(); + // get len after inserting a list + let after_len = self.bytes.len(); - // diff is list len - let list_len = after_len - before_len; - self.insert_list_len_at_pos(list_len, before_len); - } + // diff is list len + let list_len = after_len - before_len; + self.insert_list_len_at_pos(list_len, before_len); + } } #[cfg(test)] mod tests { - use std::{fmt, cmp}; - use std::str::FromStr; - use rlp; - use rlp::{Rlp, RlpStream, Decodable}; - use uint::U256; + use std::{fmt, cmp}; + use std::str::FromStr; + use rlp; + use rlp::{Rlp, RlpStream, Decodable}; + use uint::U256; - #[test] - fn rlp_at() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - { - let rlp = Rlp::new(&data); - assert!(rlp.is_list()); - let animals = as rlp::Decodable>::decode(&rlp).unwrap(); - assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); - - let cat = rlp.at(0).unwrap(); - assert!(cat.is_value()); - assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode(&cat).unwrap(), "cat".to_string()); - - let dog = rlp.at(1).unwrap(); - assert!(dog.is_value()); - assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); - assert_eq!(String::decode(&dog).unwrap(), "dog".to_string()); + #[test] + fn rlp_at() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = Rlp::new(&data); + assert!(rlp.is_list()); + let animals = + as rlp::Decodable>::decode(&rlp).unwrap(); + assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); - let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_value()); - assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode(&cat_again).unwrap(), "cat".to_string()); - } - } + let cat = rlp.at(0).unwrap(); + assert!(cat.is_value()); + assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(String::decode(&cat).unwrap(), "cat".to_string()); - #[test] - fn rlp_at_err() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; - { - let rlp = Rlp::new(&data); - assert!(rlp.is_list()); + let dog = rlp.at(1).unwrap(); + assert!(dog.is_value()); + assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + assert_eq!(String::decode(&dog).unwrap(), "dog".to_string()); - let cat_err = rlp.at(0).unwrap_err(); - assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_value()); + assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(String::decode(&cat_again).unwrap(), "cat".to_string()); + } + } - let dog_err = rlp.at(1).unwrap_err(); - assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); - } - } + #[test] + fn rlp_at_err() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; + { + let rlp = Rlp::new(&data); + assert!(rlp.is_list()); - #[test] - fn rlp_iter() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - { - let rlp = Rlp::new(&data); - let mut iter = rlp.iter(); + let cat_err = rlp.at(0).unwrap_err(); + assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); - let cat = iter.next().unwrap(); - assert!(cat.is_value()); - assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + let dog_err = rlp.at(1).unwrap_err(); + assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); + } + } - let dog = iter.next().unwrap(); - assert!(dog.is_value()); - assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + #[test] + fn rlp_iter() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = Rlp::new(&data); + let mut iter = rlp.iter(); - let none = iter.next(); - assert!(none.is_none()); + let cat = iter.next().unwrap(); + assert!(cat.is_value()); + assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); - let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_value()); - assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); - } - } + let dog = iter.next().unwrap(); + assert!(dog.is_value()); + assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); - struct ETestPair(T, Vec) where T: rlp::Encodable; + let none = iter.next(); + assert!(none.is_none()); - fn run_encode_tests(tests: Vec>) where T: rlp::Encodable { - for t in &tests { - let res = rlp::encode(&t.0); - assert_eq!(res, &t.1[..]); - } - } + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_value()); + assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + } + } - #[test] - fn encode_u8() { - let tests = vec![ + struct ETestPair(T, Vec) where T: rlp::Encodable; + + fn run_encode_tests(tests: Vec>) + where T: rlp::Encodable + { + for t in &tests { + let res = rlp::encode(&t.0); + assert_eq!(res, &t.1[..]); + } + } + + #[test] + fn encode_u8() { + let tests = vec![ ETestPair(0u8, vec![0x80u8]), ETestPair(15, vec![15]), ETestPair(55, vec![55]), @@ -606,139 +645,139 @@ mod tests { ETestPair(0x80, vec![0x81, 0x80]), ETestPair(0xff, vec![0x81, 0xff]), ]; - run_encode_tests(tests); - } + run_encode_tests(tests); + } - #[test] - fn encode_u16() { - let tests = vec![ + #[test] + fn encode_u16() { + let tests = vec![ ETestPair(0u16, vec![0x80u8]), ETestPair(0x100, vec![0x82, 0x01, 0x00]), ETestPair(0xffff, vec![0x82, 0xff, 0xff]), ]; - run_encode_tests(tests); - } + run_encode_tests(tests); + } - #[test] - fn encode_u32() { - let tests = vec![ + #[test] + fn encode_u32() { + let tests = vec![ ETestPair(0u32, vec![0x80u8]), ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), ]; - run_encode_tests(tests); - } + run_encode_tests(tests); + } - #[test] - fn encode_u64() { - let tests = vec![ + #[test] + fn encode_u64() { + let tests = vec![ ETestPair(0u64, vec![0x80u8]), ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), ]; - run_encode_tests(tests); - } + run_encode_tests(tests); + } - #[test] - fn encode_u256() { - let tests = vec![ - ETestPair(U256::from(0u64), vec![0x80u8]), - ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(U256::from(0xffffffffu64), vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]) - ]; - run_encode_tests(tests); - } + #[test] + fn encode_u256() { + let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), + ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_encode_tests(tests); + } - #[test] - fn encode_str() { - let tests = vec![ - ETestPair("cat", vec![0x83, b'c', b'a', b't']), - ETestPair("dog", vec![0x83, b'd', b'o', b'g']), - ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), - ETestPair("", vec![0x80]), - ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', b'r', - b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', b't', b',', - b' ', b'c', b'o', b'n', b's', b'e', b'c', b't', b'e', b't', - b'u', b'r', b' ', b'a', b'd', b'i', b'p', b'i', b's', b'i', - b'c', b'i', b'n', b'g', b' ', b'e', b'l', b'i', b't']) - ]; - run_encode_tests(tests); - } + #[test] + fn encode_str() { + let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), + ETestPair("dog", vec![0x83, b'd', b'o', b'g']), + ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), + ETestPair("", vec![0x80]), + ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_encode_tests(tests); + } - #[test] - fn encode_vector_u8() { - let tests = vec![ + #[test] + fn encode_vector_u8() { + let tests = vec![ ETestPair(vec![], vec![0xc0]), ETestPair(vec![15u8], vec![0xc1, 0x0f]), ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_encode_tests(tests); - } + run_encode_tests(tests); + } - #[test] - fn encode_vector_u64() { - let tests = vec![ + #[test] + fn encode_vector_u64() { + let tests = vec![ ETestPair(vec![], vec![0xc0]), ETestPair(vec![15u64], vec![0xc1, 0x0f]), ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_encode_tests(tests); - } + run_encode_tests(tests); + } - #[test] - fn encode_vector_str() { - let tests = vec![ - ETestPair(vec!["cat", "dog"], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']) - ]; - run_encode_tests(tests); - } + #[test] + fn encode_vector_str() { + let tests = vec![ETestPair(vec!["cat", "dog"], + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_encode_tests(tests); + } - #[test] - fn encode_vector_of_vectors_str() { - let tests = vec![ - ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't']) - ]; - run_encode_tests(tests); - } + #[test] + fn encode_vector_of_vectors_str() { + let tests = vec![ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_encode_tests(tests); + } - #[test] - fn rlp_stream() { - let mut stream = RlpStream::new_list(2); - stream.append(&"cat").append(&"dog"); - let out = stream.out().unwrap(); - assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - } + #[test] + fn rlp_stream() { + let mut stream = RlpStream::new_list(2); + stream.append(&"cat").append(&"dog"); + let out = stream.out().unwrap(); + assert_eq!(out, + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + } - #[test] - fn rlp_stream_list() { - let mut stream = RlpStream::new_list(3); - stream.append_list(0); - stream.append_list(1).append_list(0); - stream.append_list(2).append_list(0).append_list(1).append_list(0); - let out = stream.out().unwrap(); - assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); - } + #[test] + fn rlp_stream_list() { + let mut stream = RlpStream::new_list(3); + stream.append_list(0); + stream.append_list(1).append_list(0); + stream.append_list(2).append_list(0).append_list(1).append_list(0); + let out = stream.out().unwrap(); + assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); + } - struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; + struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; - fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { - for t in &tests { - let res: T = rlp::decode(&t.1).unwrap(); - assert_eq!(res, t.0); - } - } + fn run_decode_tests(tests: Vec>) + where T: rlp::Decodable + fmt::Debug + cmp::Eq + { + for t in &tests { + let res: T = rlp::decode(&t.1).unwrap(); + assert_eq!(res, t.0); + } + } - #[test] - fn decode_u8() { - let tests = vec![ + #[test] + fn decode_u8() { + let tests = vec![ DTestPair(0u8, vec![0u8]), DTestPair(15, vec![15]), DTestPair(55, vec![55]), @@ -747,107 +786,106 @@ mod tests { DTestPair(0x80, vec![0x81, 0x80]), DTestPair(0xff, vec![0x81, 0xff]), ]; - run_decode_tests(tests); - } + run_decode_tests(tests); + } - #[test] - fn decode_u16() { - let tests = vec![ + #[test] + fn decode_u16() { + let tests = vec![ DTestPair(0u16, vec![0u8]), DTestPair(0x100, vec![0x82, 0x01, 0x00]), DTestPair(0xffff, vec![0x82, 0xff, 0xff]), ]; - run_decode_tests(tests); - } + run_decode_tests(tests); + } - #[test] - fn decode_u32() { - let tests = vec![ + #[test] + fn decode_u32() { + let tests = vec![ DTestPair(0u32, vec![0u8]), DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), ]; - run_decode_tests(tests); - } + run_decode_tests(tests); + } - #[test] - fn decode_u64() { - let tests = vec![ + #[test] + fn decode_u64() { + let tests = vec![ DTestPair(0u64, vec![0u8]), DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), ]; - run_decode_tests(tests); - } + run_decode_tests(tests); + } - #[test] - fn decode_u256() { - let tests = vec![ - DTestPair(U256::from(0u64), vec![0x80u8]), - DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(U256::from(0xffffffffu64), vec![0x84, 0xff, 0xff, 0xff, 0xff]), - DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]) - ]; - run_decode_tests(tests); - } + #[test] + fn decode_u256() { + let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), + DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_decode_tests(tests); + } - #[test] - fn decode_str() { - let tests = vec![ - DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), - DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), - DTestPair("Marek".to_string(), vec![0x85, b'M', b'a', b'r', b'e', b'k']), - DTestPair("".to_string(), vec![0x80]), - DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit".to_string(), - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', b'r', - b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', b't', b',', - b' ', b'c', b'o', b'n', b's', b'e', b'c', b't', b'e', b't', - b'u', b'r', b' ', b'a', b'd', b'i', b'p', b'i', b's', b'i', - b'c', b'i', b'n', b'g', b' ', b'e', b'l', b'i', b't']) - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_vector_u8() { - let tests = vec![ + #[test] + fn decode_str() { + let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), + DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), + DTestPair("Marek".to_string(), + vec![0x85, b'M', b'a', b'r', b'e', b'k']), + DTestPair("".to_string(), vec![0x80]), + DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" + .to_string(), + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_decode_tests(tests); + } + + #[test] + fn decode_vector_u8() { + let tests = vec![ DTestPair(vec![] as Vec, vec![0xc0]), DTestPair(vec![15u8], vec![0xc1, 0x0f]), DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_decode_tests(tests); - } + run_decode_tests(tests); + } - #[test] - fn decode_vector_u64() { - let tests = vec![ + #[test] + fn decode_vector_u64() { + let tests = vec![ DTestPair(vec![], vec![0xc0]), DTestPair(vec![15u64], vec![0xc1, 0x0f]), DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_decode_tests(tests); - } + run_decode_tests(tests); + } - #[test] - fn decode_vector_str() { - let tests = vec![ - DTestPair(vec!["cat".to_string(), "dog".to_string()], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']) - ]; - run_decode_tests(tests); - } + #[test] + fn decode_vector_str() { + let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_decode_tests(tests); + } - #[test] - fn decode_vector_of_vectors_str() { - let tests = vec![ - DTestPair(vec![vec!["cat".to_string()]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't']) - ]; - run_decode_tests(tests); - } + #[test] + fn decode_vector_of_vectors_str() { + let tests = vec![DTestPair(vec![vec!["cat".to_string()]], + vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_decode_tests(tests); + } } - From 815e01a781171d250fcd86f93a411a993bcf8787 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 27 Nov 2015 23:24:01 +0100 Subject: [PATCH 052/379] removed copy from Hash, improved Clone, added BitOr --- src/hash.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index fc89b4ab9..98a76ad87 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,7 +1,7 @@ use std::str::FromStr; use std::fmt; use std::hash::{Hash, Hasher}; -use std::ops::{Index, IndexMut}; +use std::ops::{Index, IndexMut, BitOr}; use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; @@ -80,10 +80,14 @@ macro_rules! impl_hash { impl Clone for $from { fn clone(&self) -> $from { - *self + unsafe { + use std::{mem, ptr}; + let mut ret: $from = mem::uninitialized(); + ptr::copy(self.0.as_ptr(), ret.0.as_mut_ptr(), mem::size_of::<$from>()); + ret + } } } - impl Copy for $from {} impl PartialEq for $from { fn eq(&self, other: &Self) -> bool { @@ -115,6 +119,30 @@ macro_rules! impl_hash { &mut self.0[index] } } + + impl<'a> BitOr for &'a $from { + type Output = $from; + + fn bitor(self, rhs: Self) -> Self::Output { + unsafe { + use std::mem; + let mut ret: $from = mem::uninitialized(); + for i in 0..$size { + ret.0[i] = self.0[i] | rhs.0[i]; + } + ret + } + } + } + + impl BitOr for $from { + type Output = $from; + + fn bitor(self, rhs: Self) -> Self::Output { + &self | &rhs + } + } + } } @@ -126,7 +154,6 @@ impl_hash!(H512, 64); impl_hash!(H520, 65); impl_hash!(H1024, 128); impl_hash!(H2048, 256); -impl_hash!(H4096, 512); #[test] fn hash() { @@ -138,3 +165,16 @@ fn hash() { assert!(h != H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); assert!(h != H64([0; 8])); } + +#[test] +fn hash_bitor() { + let a = H64([1; 8]); + let b = H64([2; 8]); + let c = H64([3; 8]); + + // borrow + assert_eq!(&a | &b, c); + + // move + assert_eq!(a | b, c); +} From 2db779a63bad0712c027f4d226055e71ab9384fd Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 00:14:40 +0100 Subject: [PATCH 053/379] Memory DB. --- src/bytes.rs | 2 + src/hashdb.rs | 9 +++ src/lib.rs | 14 ++-- src/memorydb.rs | 168 ++++++++++++++++++++++++++++++++++++++++++++++++ src/sha3.rs | 2 +- 5 files changed, 188 insertions(+), 7 deletions(-) create mode 100644 src/hashdb.rs create mode 100644 src/memorydb.rs diff --git a/src/bytes.rs b/src/bytes.rs index 870c9eef3..91f9fcda2 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -10,6 +10,8 @@ use std::fmt; use std::error::Error as StdError; use uint::{U128, U256}; +pub type Bytes = Vec; + pub trait BytesConvertable { fn bytes(&self) -> &[u8]; } diff --git a/src/hashdb.rs b/src/hashdb.rs new file mode 100644 index 000000000..b686da78d --- /dev/null +++ b/src/hashdb.rs @@ -0,0 +1,9 @@ +use hash::*; +use bytes::Bytes; + +pub trait HashDB { + fn lookup(&self, key: &H256) -> Option<&Bytes>; + fn exists(&self, key: &H256) -> bool; + fn insert(&mut self, value: &[u8]) -> H256; + fn kill(&mut self, key: &H256); +} diff --git a/src/lib.rs b/src/lib.rs index 7c480c02b..a43585e0c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,8 +6,6 @@ extern crate tiny_keccak; #[macro_use] extern crate log; -pub use std::str::FromStr; - pub mod error; pub mod hash; pub mod uint; @@ -16,11 +14,15 @@ pub mod rlp; pub mod vector; pub mod db; pub mod sha3; +pub mod hashdb; +pub mod memorydb; //pub mod network; -pub type Bytes = Vec; +pub use std::str::FromStr; +pub use hash::*; +pub use sha3::*; +pub use bytes::*; +pub use hashdb::*; +pub use memorydb::*; -#[test] -fn it_works() { -} diff --git a/src/memorydb.rs b/src/memorydb.rs new file mode 100644 index 000000000..d732b9e5f --- /dev/null +++ b/src/memorydb.rs @@ -0,0 +1,168 @@ +//! Reference-counted memory-based HashDB implementation. +//! +//! # Example +//! ```rust +//! extern crate ethcore_util; +//! use ethcore_util::hashdb::*; +//! use ethcore_util::memorydb::*; +//! fn main() { +//! let mut m = MemoryDB::new(); +//! let d = "Hello world!".as_bytes(); +//! +//! let k = m.insert(d); +//! assert!(m.exists(&k)); +//! assert_eq!(m.lookup(&k).unwrap(), &d); +//! +//! m.insert(d); +//! assert!(m.exists(&k)); +//! +//! m.kill(&k); +//! assert!(m.exists(&k)); +//! +//! m.kill(&k); +//! assert!(!m.exists(&k)); +//! +//! m.insert(d); +//! assert!(m.exists(&k)); +//! assert_eq!(m.lookup(&k).unwrap(), &d); +//! +//! m.kill(&k); +//! assert!(!m.exists(&k)); +//! } + +//! ``` +use hash::*; +use bytes::*; +use sha3::*; +use hashdb::*; +use std::collections::HashMap; + +#[derive(Debug,Clone)] +pub struct MemoryDB { + data: HashMap, +} + +impl MemoryDB { + pub fn new() -> MemoryDB { + MemoryDB { + data: HashMap::new() + } + } + + pub fn clear(&mut self) { + self.data.clear(); + } + + pub fn purge(&mut self) { + let empties: Vec<_> = self.data.iter().filter(|&(_, &(_, rc))| rc <= 0).map(|(k, _)| k.clone()).collect(); + for empty in empties { self.data.remove(&empty); } + } +} + +impl HashDB for MemoryDB { + /// Do a hash dereference and look up a given hash into the bytes that make it up, + /// returning None if nothing is found (or if the only entries have 0 or fewer + /// references). + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let hello_bytes = "Hello world!".as_bytes(); + /// let hash = m.insert(hello_bytes); + /// assert_eq!(m.lookup(&hash).unwrap(), &hello_bytes); + /// } + /// ``` + fn lookup(&self, key: &H256) -> Option<&Bytes> { + match self.data.get(key) { + Some(&(ref d, rc)) if rc > 0 => Some(d), + _ => None + } + } + + /// Check for the existance of a hash-key. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// use ethcore_util::sha3::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let hello_bytes = "Hello world!".as_bytes(); + /// assert!(!m.exists(&hello_bytes.sha3())); + /// let key = m.insert(hello_bytes); + /// assert!(m.exists(&key)); + /// m.kill(&key); + /// assert!(!m.exists(&key)); + /// } + /// ``` + fn exists(&self, key: &H256) -> bool { + match self.data.get(key) { + Some(&(_, x)) if x > 0 => true, + _ => false + } + } + + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions + /// are counted and the equivalent number of `kill()`s must be performed before the data + /// is considered dead. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// use ethcore_util::hash::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let key = m.insert("Hello world!".as_bytes()); + /// assert!(m.exists(&key)); + /// } + /// ``` + fn insert(&mut self, value: &[u8]) -> H256 { + let key = value.sha3(); + if match self.data.get_mut(&key) { + Some(&mut (ref mut old_value, ref mut rc @ 0)) => { *old_value = From::from(value.bytes()); *rc = 1; false }, + Some(&mut (_, ref mut x)) => { *x += 1; false } , + None => true, + }{ // ... None falls through into... + self.data.insert(key, (From::from(value.bytes()), 1)); + } + key + } + /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of inserts may + /// happen without the data being eventually being inserted into the DB. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// use ethcore_util::sha3::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let d = "Hello world!".as_bytes(); + /// let key = &d.sha3(); + /// m.kill(key); // OK - we now owe an insertion. + /// assert!(!m.exists(key)); + /// m.insert(d); // OK - now it's "empty" again. + /// assert!(!m.exists(key)); + /// m.insert(d); // OK - now we've + /// assert_eq!(m.lookup(key).unwrap(), &d); + /// } + /// ``` + fn kill(&mut self, key: &H256) { + if match self.data.get_mut(key) { + Some(&mut (_, ref mut x)) => { *x -= 1; false } + None => true + }{ // ... None falls through into... + self.data.insert(*key, (Bytes::new(), -1)); + } + } +} + diff --git a/src/sha3.rs b/src/sha3.rs index e2866d288..e4ba1b007 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -3,7 +3,7 @@ use tiny_keccak::keccak_256; use bytes::BytesConvertable; use hash::H256; -trait Hashable { +pub trait Hashable { fn sha3(&self) -> H256; } From b7e69e04fe4c8d1c73b0811769811b4c7437fb97 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 01:12:59 +0100 Subject: [PATCH 054/379] More documentation. --- src/memorydb.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/memorydb.rs b/src/memorydb.rs index d732b9e5f..a76007a86 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -43,18 +43,39 @@ pub struct MemoryDB { } impl MemoryDB { + /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { MemoryDB { data: HashMap::new() } } + /// Clear all data from the database. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let hello_bytes = "Hello world!".as_bytes(); + /// let hash = m.insert(hello_bytes); + /// assert!(m.exists(&hash)); + /// m.clear(); + /// assert!(!m.exists(&hash)); + /// } + /// ``` pub fn clear(&mut self) { self.data.clear(); } + /// Purge all zero-referenced data from the database. pub fn purge(&mut self) { - let empties: Vec<_> = self.data.iter().filter(|&(_, &(_, rc))| rc <= 0).map(|(k, _)| k.clone()).collect(); + let empties: Vec<_> = self.data.iter() + .filter(|&(_, &(_, rc))| rc == 0) + .map(|(k, _)| k.clone()) + .collect(); for empty in empties { self.data.remove(&empty); } } } From 47f8695c3b1f4e688baa64565baedc0b9aeea7ee Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 01:38:00 +0100 Subject: [PATCH 055/379] common changes in hash --- src/bloom.rs | 7 ------- src/hash.rs | 34 ++++++++++++++++++++++++++-------- src/lib.rs | 1 - src/sha3.rs | 2 +- 4 files changed, 27 insertions(+), 17 deletions(-) delete mode 100644 src/bloom.rs diff --git a/src/bloom.rs b/src/bloom.rs deleted file mode 100644 index fe619fae6..000000000 --- a/src/bloom.rs +++ /dev/null @@ -1,7 +0,0 @@ -use bytes::BytesConvertable; -// use hash::FixedHash; - -pub trait Bloomable { - fn shift_bloom(&mut self, bytes: &T) where T: BytesConvertable; - fn contains_bloom(&self, bytes: &T) -> bool where T: BytesConvertable; -} diff --git a/src/hash.rs b/src/hash.rs index 98a76ad87..310a11dbb 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -9,10 +9,13 @@ use rand::os::OsRng; use bytes::BytesConvertable; /// types implementing FixedHash must be also BytesConvertable -pub trait FixedHash: BytesConvertable { +pub trait FixedHash: Sized + BytesConvertable { + fn new() -> Self; fn random() -> Self; fn randomize(&mut self); fn mut_bytes(&mut self) -> &mut [u8]; + fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; + fn bloom_part(&self) -> T where T: FixedHash; } macro_rules! impl_hash { @@ -20,13 +23,6 @@ macro_rules! impl_hash { #[derive(Eq)] pub struct $from (pub [u8; $size]); - - impl $from { - fn new() -> $from { - $from([0; $size]) - } - } - impl BytesConvertable for $from { fn bytes(&self) -> &[u8] { &self.0 @@ -34,6 +30,10 @@ macro_rules! impl_hash { } impl FixedHash for $from { + fn new() -> $from { + $from([0; $size]) + } + fn random() -> $from { let mut hash = $from::new(); hash.randomize(); @@ -48,6 +48,24 @@ macro_rules! impl_hash { fn mut_bytes(&mut self) -> &mut [u8] { &mut self.0 } + + fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash { + let bp: Self = b.bloom_part(); + let new_self = &bp | self; + + // impl |= instead + + unsafe { + use std::{mem, ptr}; + ptr::copy(new_self.0.as_ptr(), self.0.as_mut_ptr(), mem::size_of::()); + } + + self + } + + fn bloom_part(&self) -> T where T: FixedHash { + panic!() + } } impl FromStr for $from { diff --git a/src/lib.rs b/src/lib.rs index a52e35925..7c480c02b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,6 @@ pub mod rlp; pub mod vector; pub mod db; pub mod sha3; -pub mod bloom; //pub mod network; diff --git a/src/sha3.rs b/src/sha3.rs index b7eec2a5f..ee328913c 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -3,7 +3,7 @@ use tiny_keccak::keccak_256; use bytes::BytesConvertable; use hash::{FixedHash, H256}; -trait Hashable { +pub trait Hashable { fn sha3(&self) -> H256; } From 705ab53948d7512b5eaa58390c464f75dd247f05 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 01:38:36 +0100 Subject: [PATCH 056/379] Fix for latest changes. --- src/hashdb.rs | 73 ++++++++++++++++++++++++ src/memorydb.rs | 149 ++++++++++++++---------------------------------- 2 files changed, 117 insertions(+), 105 deletions(-) diff --git a/src/hashdb.rs b/src/hashdb.rs index b686da78d..d023fe70d 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -2,8 +2,81 @@ use hash::*; use bytes::Bytes; pub trait HashDB { + /// Look up a given hash into the bytes that hash to it, returning None if the + /// hash is not known. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let hello_bytes = "Hello world!".as_bytes(); + /// let hash = m.insert(hello_bytes); + /// assert_eq!(m.lookup(&hash).unwrap(), &hello_bytes); + /// } + /// ``` fn lookup(&self, key: &H256) -> Option<&Bytes>; + + /// Check for the existance of a hash-key. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// use ethcore_util::sha3::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let hello_bytes = "Hello world!".as_bytes(); + /// assert!(!m.exists(&hello_bytes.sha3())); + /// let key = m.insert(hello_bytes); + /// assert!(m.exists(&key)); + /// m.kill(&key); + /// assert!(!m.exists(&key)); + /// } + /// ``` fn exists(&self, key: &H256) -> bool; + + /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions + /// are counted and the equivalent number of `kill()`s must be performed before the data + /// is considered dead. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// use ethcore_util::hash::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let key = m.insert("Hello world!".as_bytes()); + /// assert!(m.exists(&key)); + /// } + /// ``` fn insert(&mut self, value: &[u8]) -> H256; + + /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may + /// happen without the data being eventually being inserted into the DB. + /// + /// # Examples + /// ```rust + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::memorydb::*; + /// use ethcore_util::sha3::*; + /// fn main() { + /// let mut m = MemoryDB::new(); + /// let d = "Hello world!".as_bytes(); + /// let key = &d.sha3(); + /// m.kill(key); // OK - we now owe an insertion. + /// assert!(!m.exists(key)); + /// m.insert(d); // OK - now it's "empty" again. + /// assert!(!m.exists(key)); + /// m.insert(d); // OK - now we've + /// assert_eq!(m.lookup(key).unwrap(), &d); + /// } + /// ``` fn kill(&mut self, key: &H256); } diff --git a/src/memorydb.rs b/src/memorydb.rs index a76007a86..55fdc12da 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -1,36 +1,5 @@ //! Reference-counted memory-based HashDB implementation. -//! -//! # Example -//! ```rust -//! extern crate ethcore_util; -//! use ethcore_util::hashdb::*; -//! use ethcore_util::memorydb::*; -//! fn main() { -//! let mut m = MemoryDB::new(); -//! let d = "Hello world!".as_bytes(); -//! -//! let k = m.insert(d); -//! assert!(m.exists(&k)); -//! assert_eq!(m.lookup(&k).unwrap(), &d); -//! -//! m.insert(d); -//! assert!(m.exists(&k)); -//! -//! m.kill(&k); -//! assert!(m.exists(&k)); -//! -//! m.kill(&k); -//! assert!(!m.exists(&k)); -//! -//! m.insert(d); -//! assert!(m.exists(&k)); -//! assert_eq!(m.lookup(&k).unwrap(), &d); -//! -//! m.kill(&k); -//! assert!(!m.exists(&k)); -//! } -//! ``` use hash::*; use bytes::*; use sha3::*; @@ -38,6 +7,43 @@ use hashdb::*; use std::collections::HashMap; #[derive(Debug,Clone)] +/// Reference-counted memory-based HashDB implementation. +/// +/// Use `new()` to create a new database. Insert items with `insert()`, remove items +/// with `kill()`, check for existance with `exists()` and lookup a hash to derive +/// the data with `lookup()`. Clear with `clear()` and purge the portions of the data +/// that have no references with `purge()`. +/// +/// # Example +/// ```rust +/// extern crate ethcore_util; +/// use ethcore_util::hashdb::*; +/// use ethcore_util::memorydb::*; +/// fn main() { +/// let mut m = MemoryDB::new(); +/// let d = "Hello world!".as_bytes(); +/// +/// let k = m.insert(d); +/// assert!(m.exists(&k)); +/// assert_eq!(m.lookup(&k).unwrap(), &d); +/// +/// m.insert(d); +/// assert!(m.exists(&k)); +/// +/// m.kill(&k); +/// assert!(m.exists(&k)); +/// +/// m.kill(&k); +/// assert!(!m.exists(&k)); +/// +/// m.insert(d); +/// assert!(m.exists(&k)); +/// assert_eq!(m.lookup(&k).unwrap(), &d); +/// +/// m.kill(&k); +/// assert!(!m.exists(&k)); +/// } +/// ``` pub struct MemoryDB { data: HashMap, } @@ -81,22 +87,6 @@ impl MemoryDB { } impl HashDB for MemoryDB { - /// Do a hash dereference and look up a given hash into the bytes that make it up, - /// returning None if nothing is found (or if the only entries have 0 or fewer - /// references). - /// - /// # Examples - /// ```rust - /// extern crate ethcore_util; - /// use ethcore_util::hashdb::*; - /// use ethcore_util::memorydb::*; - /// fn main() { - /// let mut m = MemoryDB::new(); - /// let hello_bytes = "Hello world!".as_bytes(); - /// let hash = m.insert(hello_bytes); - /// assert_eq!(m.lookup(&hash).unwrap(), &hello_bytes); - /// } - /// ``` fn lookup(&self, key: &H256) -> Option<&Bytes> { match self.data.get(key) { Some(&(ref d, rc)) if rc > 0 => Some(d), @@ -104,24 +94,6 @@ impl HashDB for MemoryDB { } } - /// Check for the existance of a hash-key. - /// - /// # Examples - /// ```rust - /// extern crate ethcore_util; - /// use ethcore_util::hashdb::*; - /// use ethcore_util::memorydb::*; - /// use ethcore_util::sha3::*; - /// fn main() { - /// let mut m = MemoryDB::new(); - /// let hello_bytes = "Hello world!".as_bytes(); - /// assert!(!m.exists(&hello_bytes.sha3())); - /// let key = m.insert(hello_bytes); - /// assert!(m.exists(&key)); - /// m.kill(&key); - /// assert!(!m.exists(&key)); - /// } - /// ``` fn exists(&self, key: &H256) -> bool { match self.data.get(key) { Some(&(_, x)) if x > 0 => true, @@ -129,60 +101,27 @@ impl HashDB for MemoryDB { } } - /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions - /// are counted and the equivalent number of `kill()`s must be performed before the data - /// is considered dead. - /// - /// # Examples - /// ```rust - /// extern crate ethcore_util; - /// use ethcore_util::hashdb::*; - /// use ethcore_util::memorydb::*; - /// use ethcore_util::hash::*; - /// fn main() { - /// let mut m = MemoryDB::new(); - /// let key = m.insert("Hello world!".as_bytes()); - /// assert!(m.exists(&key)); - /// } - /// ``` fn insert(&mut self, value: &[u8]) -> H256 { let key = value.sha3(); if match self.data.get_mut(&key) { - Some(&mut (ref mut old_value, ref mut rc @ 0)) => { *old_value = From::from(value.bytes()); *rc = 1; false }, + Some(&mut (ref mut old_value, ref mut rc @ 0)) => { + *old_value = From::from(value.bytes()); + *rc = 1; + false + }, Some(&mut (_, ref mut x)) => { *x += 1; false } , None => true, }{ // ... None falls through into... - self.data.insert(key, (From::from(value.bytes()), 1)); + self.data.insert(key.clone(), (From::from(value.bytes()), 1)); } key } - /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of inserts may - /// happen without the data being eventually being inserted into the DB. - /// - /// # Examples - /// ```rust - /// extern crate ethcore_util; - /// use ethcore_util::hashdb::*; - /// use ethcore_util::memorydb::*; - /// use ethcore_util::sha3::*; - /// fn main() { - /// let mut m = MemoryDB::new(); - /// let d = "Hello world!".as_bytes(); - /// let key = &d.sha3(); - /// m.kill(key); // OK - we now owe an insertion. - /// assert!(!m.exists(key)); - /// m.insert(d); // OK - now it's "empty" again. - /// assert!(!m.exists(key)); - /// m.insert(d); // OK - now we've - /// assert_eq!(m.lookup(key).unwrap(), &d); - /// } - /// ``` fn kill(&mut self, key: &H256) { if match self.data.get_mut(key) { Some(&mut (_, ref mut x)) => { *x -= 1; false } None => true }{ // ... None falls through into... - self.data.insert(*key, (Bytes::new(), -1)); + self.data.insert(key.clone(), (Bytes::new(), -1)); } } } From 33eeb8f47773b1979bd2f83844459255ceeef565 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 03:02:06 +0100 Subject: [PATCH 057/379] math.rs, log2, FixedHash::shift_bloom, FixedHash::contains_bloom --- src/hash.rs | 147 ++++++++++++++++++++++++++++++++++++++++++---------- src/lib.rs | 1 + src/math.rs | 10 ++++ 3 files changed, 132 insertions(+), 26 deletions(-) create mode 100644 src/math.rs diff --git a/src/hash.rs b/src/hash.rs index 310a11dbb..8612cf226 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,12 +1,13 @@ use std::str::FromStr; use std::fmt; use std::hash::{Hash, Hasher}; -use std::ops::{Index, IndexMut, BitOr}; +use std::ops::{Index, IndexMut, BitOr, BitAnd}; use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; use rand::os::OsRng; use bytes::BytesConvertable; +use math::log2; /// types implementing FixedHash must be also BytesConvertable pub trait FixedHash: Sized + BytesConvertable { @@ -15,7 +16,8 @@ pub trait FixedHash: Sized + BytesConvertable { fn randomize(&mut self); fn mut_bytes(&mut self) -> &mut [u8]; fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; - fn bloom_part(&self) -> T where T: FixedHash; + fn bloom_part(&self, m: usize) -> T where T: FixedHash; + fn contains_bloom(&self, b: &T) -> bool where T: FixedHash; } macro_rules! impl_hash { @@ -50,7 +52,7 @@ macro_rules! impl_hash { } fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash { - let bp: Self = b.bloom_part(); + let bp: Self = b.bloom_part($size); let new_self = &bp | self; // impl |= instead @@ -63,8 +65,45 @@ macro_rules! impl_hash { self } - fn bloom_part(&self) -> T where T: FixedHash { - panic!() + fn bloom_part(&self, m: usize) -> T where T: FixedHash { + // numbers of bits + // TODO: move it to some constant + let p = 3; + + let bloom_bits = m * 8; + let mask = bloom_bits - 1; + let bloom_bytes = (log2(bloom_bits) + 7) / 8; + //println!("bb: {}", bloom_bytes); + + // must be a power of 2 + assert_eq!(m & (m - 1), 0); + // out of range + assert!(p * bloom_bytes <= $size); + + // return type + let mut ret = T::new(); + + // 'ptr' to out slice + let mut ptr = 0; + + // set p number of bits, + // p is equal 3 according to yellowpaper + for _ in 0..p { + let mut index = 0 as usize; + for _ in 0..bloom_bytes { + index = (index << 8) | self.0[ptr] as usize; + ptr += 1; + } + index &= mask; + ret.mut_bytes()[m - 1 - index / 8] |= 1 << (index % 8); + } + + ret + } + + fn contains_bloom(&self, b: &T) -> bool where T: FixedHash { + let bp: Self = b.bloom_part($size); + (&bp & self) == bp } } @@ -138,6 +177,7 @@ macro_rules! impl_hash { } } + /// BitOr on references impl<'a> BitOr for &'a $from { type Output = $from; @@ -153,6 +193,7 @@ macro_rules! impl_hash { } } + /// Moving BitOr impl BitOr for $from { type Output = $from; @@ -161,6 +202,31 @@ macro_rules! impl_hash { } } + /// BitAnd on references + impl <'a> BitAnd for &'a $from { + type Output = $from; + + fn bitand(self, rhs: Self) -> Self::Output { + unsafe { + use std::mem; + let mut ret: $from = mem::uninitialized(); + for i in 0..$size { + ret.0[i] = self.0[i] & rhs.0[i]; + } + ret + } + } + } + + /// Moving BitAnd + impl BitAnd for $from { + type Output = $from; + + fn bitand(self, rhs: Self) -> Self::Output { + &self & &rhs + } + } + } } @@ -173,26 +239,55 @@ impl_hash!(H520, 65); impl_hash!(H1024, 128); impl_hash!(H2048, 256); -#[test] -fn hash() { - let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); - assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); - assert_eq!(format!("{}", h), "0123456789abcdef"); - assert_eq!(format!("{:?}", h), "0123456789abcdef"); - assert!(h == h); - assert!(h != H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); - assert!(h != H64([0; 8])); +#[cfg(test)] +mod tests { + use hash::*; + use std::str::FromStr; + + #[test] + fn hash() { + let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); + assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); + assert_eq!(format!("{}", h), "0123456789abcdef"); + assert_eq!(format!("{:?}", h), "0123456789abcdef"); + assert!(h == h); + assert!(h != H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); + assert!(h != H64([0; 8])); + } + + #[test] + fn hash_bitor() { + let a = H64([1; 8]); + let b = H64([2; 8]); + let c = H64([3; 8]); + + // borrow + assert_eq!(&a | &b, c); + + // move + assert_eq!(a | b, c); + } + + #[test] + fn shift_bloom() { + use sha3::Hashable; + + let bloom = H2048::from_str("00000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002020000000000000000000000000000000000000000000008000000001000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); + let address = Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(); + let topic = H256::from_str("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); + + let mut my_bloom = H2048::new(); + assert!(!my_bloom.contains_bloom(&address.sha3())); + assert!(!my_bloom.contains_bloom(&topic.sha3())); + + my_bloom.shift_bloom(&address.sha3()); + assert!(my_bloom.contains_bloom(&address.sha3())); + assert!(!my_bloom.contains_bloom(&topic.sha3())); + + my_bloom.shift_bloom(&topic.sha3()); + assert_eq!(my_bloom, bloom); + assert!(my_bloom.contains_bloom(&address.sha3())); + assert!(my_bloom.contains_bloom(&topic.sha3())); + } } -#[test] -fn hash_bitor() { - let a = H64([1; 8]); - let b = H64([2; 8]); - let c = H64([3; 8]); - - // borrow - assert_eq!(&a | &b, c); - - // move - assert_eq!(a | b, c); -} diff --git a/src/lib.rs b/src/lib.rs index a43585e0c..f94bceb86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod db; pub mod sha3; pub mod hashdb; pub mod memorydb; +pub mod math; //pub mod network; diff --git a/src/math.rs b/src/math.rs new file mode 100644 index 000000000..ba039f3d0 --- /dev/null +++ b/src/math.rs @@ -0,0 +1,10 @@ +/// log2 +pub fn log2(x: usize) -> u32 { + if x <= 1 { + return 0; + } + + let n = x.leading_zeros(); + ::std::mem::size_of::() as u32 * 8 - n +} + From 852233e4ac1b3109e2b73f876edaba68052f0c2f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 03:08:57 +0100 Subject: [PATCH 058/379] HashDB and initial OverlayDB --- src/hashdb.rs | 6 +++--- src/lib.rs | 1 + src/memorydb.rs | 24 ++++++++++++++++++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/hashdb.rs b/src/hashdb.rs index d023fe70d..48f5182bc 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -14,10 +14,10 @@ pub trait HashDB { /// let mut m = MemoryDB::new(); /// let hello_bytes = "Hello world!".as_bytes(); /// let hash = m.insert(hello_bytes); - /// assert_eq!(m.lookup(&hash).unwrap(), &hello_bytes); + /// assert_eq!(m.lookup(&hash).unwrap(), hello_bytes); /// } /// ``` - fn lookup(&self, key: &H256) -> Option<&Bytes>; + fn lookup(&self, key: &H256) -> Option; /// Check for the existance of a hash-key. /// @@ -75,7 +75,7 @@ pub trait HashDB { /// m.insert(d); // OK - now it's "empty" again. /// assert!(!m.exists(key)); /// m.insert(d); // OK - now we've - /// assert_eq!(m.lookup(key).unwrap(), &d); + /// assert_eq!(m.lookup(key).unwrap(), d); /// } /// ``` fn kill(&mut self, key: &H256); diff --git a/src/lib.rs b/src/lib.rs index f7f446ddc..cb03d00b7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,7 @@ pub mod db; pub mod sha3; pub mod hashdb; pub mod memorydb; +pub mod overlaydb; pub mod bloom; //pub mod network; diff --git a/src/memorydb.rs b/src/memorydb.rs index 55fdc12da..2f54b4524 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -25,7 +25,7 @@ use std::collections::HashMap; /// /// let k = m.insert(d); /// assert!(m.exists(&k)); -/// assert_eq!(m.lookup(&k).unwrap(), &d); +/// assert_eq!(m.lookup(&k).unwrap(), d); /// /// m.insert(d); /// assert!(m.exists(&k)); @@ -38,7 +38,7 @@ use std::collections::HashMap; /// /// m.insert(d); /// assert!(m.exists(&k)); -/// assert_eq!(m.lookup(&k).unwrap(), &d); +/// assert_eq!(m.lookup(&k).unwrap(), d); /// /// m.kill(&k); /// assert!(!m.exists(&k)); @@ -84,12 +84,27 @@ impl MemoryDB { .collect(); for empty in empties { self.data.remove(&empty); } } + + /// Grab the number of references a particular `key` has. Returns None if the key + /// doesn't exist. + fn refs(&self, key: &H256) -> Option { + self.data.get(key).map(|&(_, rc)| rc) + } + + /// Grab the value associated with a particular `key`. Returns None if the key + /// doesn't exist. + /// + /// Even when Some is returned, this is only guaranteed to return something useful + /// when `refs(key) > 0`. + fn value(&self, key: &H256) -> Option<&Bytes> { + self.data.get(key).map(|&(ref d, _)| d) + } } impl HashDB for MemoryDB { - fn lookup(&self, key: &H256) -> Option<&Bytes> { + fn lookup(&self, key: &H256) -> Option { match self.data.get(key) { - Some(&(ref d, rc)) if rc > 0 => Some(d), + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), _ => None } } @@ -116,6 +131,7 @@ impl HashDB for MemoryDB { } key } + fn kill(&mut self, key: &H256) { if match self.data.get_mut(key) { Some(&mut (_, ref mut x)) => { *x -= 1; false } From 71dc9c0ad46f79afef069026b4ba1bb34ac0e4a8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 03:09:36 +0100 Subject: [PATCH 059/379] OVerlayDB draft. --- src/overlaydb.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 src/overlaydb.rs diff --git a/src/overlaydb.rs b/src/overlaydb.rs new file mode 100644 index 000000000..99d66cd5f --- /dev/null +++ b/src/overlaydb.rs @@ -0,0 +1,70 @@ +//! Disk-backed HashDB implementation. + +use hash::*; +use bytes::*; +use sha3::*; +use hashdb::*; +use memorydb::*; +use std::ops::*; +use rocksdb::{DB, Writable}; + +#[derive(Clone)] +pub struct OverlayDB { + overlay: MemoryDB, + backing: DB, +} + +impl OverlayDB { + /// Create a new instance of OverlayDB given a `backing` database. + fn new(backing: DB) { + self.backing = backing; + overlay = MemoryDB::new(); + } + + /// Commit all memory operations to the backing database. + fn commit(&mut self) { + unimplemented!(); + } + + /// Get the refs and value of the given key. + fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { + unimplemented!(); + } +} + +impl HashDB for OverlayDB { + fn lookup(&self, key: &H256) -> Option { + // TODO: return ok if positive; if negative, check backing - might be enough references there to make + // it positive again. + let k = self.overlay.data.get(key); + match k { + Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + _ => { + let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); + match self.payload(key) { + Some((d, rc)) if rc + memrc > 0 => Some(d), + _ => None, + } + } + } + } + fn exists(&self, key: &H256) -> bool { + // TODO: copy and adapt code above. + m_overlay.exists(key) + } + fn insert(&mut self, value: &[u8]) -> H256 { m_overlay.insert(value) } + fn kill(&mut self, key: &H256) { m_overlay.kill(key); } +} + +#[test] +fn playpen() { + let mut db: DB = DB::open_default("/tmp/test").unwrap(); + db.put(b"test", b"test2"); + match db.get(b"test") { + Ok(Some(value)) => println!("Got value {:?}", value.deref()), + Ok(None) => println!("No value for that key"), + Err(e) => println!("Gah"), + } + db.delete(b"test"); + assert!(false); +} \ No newline at end of file From 111fc70d0b165324d380dc73570af93d91beb3fb Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 03:58:37 +0100 Subject: [PATCH 060/379] updated docs --- src/bytes.rs | 46 ++++++++++++++++++++++++++++++++++++++++------ src/hash.rs | 2 +- src/lib.rs | 5 +++++ 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 9521abdbd..0f82748a1 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -1,17 +1,47 @@ -//! To/From Bytes conversation for basic types -//! -//! Types implementing `ToBytes` and `FromBytes` traits -//! can be easily converted to and from bytes +//! Unified interfaces for bytes operations on basic types //! //! # Examples +//! ```rust +//! extern crate ethcore_util as util; +//! +//! fn bytes_convertable() { +//! use util::bytes::BytesConvertable; //! +//! let arr = [0; 5]; +//! let slice: &[u8] = arr.bytes(); +//! } +//! +//! fn to_bytes() { +//! use util::bytes::ToBytes; +//! +//! let a: Vec = "hello_world".to_bytes(); +//! let b: Vec = 400u32.to_bytes(); +//! let c: Vec = 0xffffffffffffffffu64.to_bytes(); +//! } +//! +//! fn from_bytes() { +//! use util::bytes::FromBytes; +//! +//! let a = String::from_bytes(&[b'd', b'o', b'g']); +//! let b = u8::from_bytes(&[0xfa]); +//! let c = u64::from_bytes(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); +//! } +//! +//! fn main() { +//! bytes_convertable(); +//! to_bytes(); +//! from_bytes(); +//! } +//! ``` use std::fmt; use std::error::Error as StdError; use uint::{U128, U256}; +/// Vector of bytes pub type Bytes = Vec; +/// Slice of bytes to underlying memory pub trait BytesConvertable { fn bytes(&self) -> &[u8]; } @@ -46,6 +76,8 @@ fn bytes_convertable() { assert_eq!([0u8; 0].bytes(), &[]); } +/// Converts given type to its shortest representation in bytes +/// /// TODO: optimise some conversations pub trait ToBytes { fn to_bytes(&self) -> Vec; @@ -141,6 +173,7 @@ macro_rules! impl_uint_to_bytes { impl_uint_to_bytes!(U256); impl_uint_to_bytes!(U128); +/// Error returned when FromBytes conversation goes wrong #[derive(Debug, PartialEq, Eq)] pub enum FromBytesError { UnexpectedEnd @@ -156,10 +189,11 @@ impl fmt::Display for FromBytesError { } } +/// Alias for the result of FromBytes trait pub type FromBytesResult = Result; -/// implements "Sized", so the compiler can deducate the size -/// of the return type +/// Converts to given type from its bytes representation +/// /// TODO: check size of bytes before conversation and return appropriate error pub trait FromBytes: Sized { fn from_bytes(bytes: &[u8]) -> FromBytesResult; diff --git a/src/hash.rs b/src/hash.rs index 8612cf226..0dfd4f487 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -23,7 +23,7 @@ pub trait FixedHash: Sized + BytesConvertable { macro_rules! impl_hash { ($from: ident, $size: expr) => { #[derive(Eq)] - pub struct $from (pub [u8; $size]); + pub struct $from ([u8; $size]); impl BytesConvertable for $from { fn bytes(&self) -> &[u8] { diff --git a/src/lib.rs b/src/lib.rs index f94bceb86..acb03afb5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,7 @@ +//! Ethcore-util library +//! +//! TODO: check reexports + extern crate rustc_serialize; extern crate mio; extern crate rand; @@ -20,6 +24,7 @@ pub mod math; //pub mod network; +// reexports pub use std::str::FromStr; pub use hash::*; pub use sha3::*; From 07f3e6d5a5cc5c7a1ae254dac9092ab2602b2ba4 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 10:50:41 +0100 Subject: [PATCH 061/379] updated tests for rlp and indention in benches --- benches/rlp.rs | 107 +++++++++++++++++++++++++------------------------ src/rlp.rs | 79 ++++++++++++++++++++++++++---------- 2 files changed, 112 insertions(+), 74 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 73af76bbe..f6a13f260 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -1,7 +1,7 @@ //! benchmarking for rlp //! should be started with: //! ```bash -//! multirust run nightly cargo bench +//! multirust run nightly cargo bench //! ``` #![feature(test)] @@ -16,80 +16,81 @@ use ethcore_util::uint::U256; #[bench] fn bench_stream_u64_value(b: &mut Bencher) { - b.iter( || { - //1029 - let mut stream = RlpStream::new(); - stream.append(&0x1023456789abcdefu64); - let _ = stream.out().unwrap(); - }); + b.iter(|| { + // u64 + let mut stream = RlpStream::new(); + stream.append(&0x1023456789abcdefu64); + let _ = stream.out().unwrap(); + }); } #[bench] fn bench_decode_u64_value(b: &mut Bencher) { - b.iter( || { - // 1029 - let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; - let rlp = Rlp::new(&data); - let _ = u64::decode(&rlp).unwrap(); - }); + b.iter(|| { + // u64 + let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + let rlp = Rlp::new(&data); + let _ = u64::decode(&rlp).unwrap(); + }); } #[bench] fn bench_stream_u256_value(b: &mut Bencher) { - b.iter( || { - //u256 - let mut stream = RlpStream::new(); - stream.append(&U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); - let _ = stream.out().unwrap(); - }); + b.iter(|| { + // u256 + let mut stream = RlpStream::new(); + stream.append(&U256::from_str("8090a0b0c0d0e0f009102030405060770000000000000001000000000\ + 00012f0") + .unwrap()); + let _ = stream.out().unwrap(); + }); } #[bench] fn bench_decode_u256_value(b: &mut Bencher) { - b.iter( || { - // u256 - let data = vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; - let rlp = Rlp::new(&data); - let _ = U256::decode(&rlp).unwrap(); - }); + b.iter(|| { + // u256 + let data = vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0x09, 0x10, 0x20, + 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; + let rlp = Rlp::new(&data); + let _ = U256::decode(&rlp).unwrap(); + }); } #[bench] fn bench_stream_nested_empty_lists(b: &mut Bencher) { - b.iter( || { - // [ [], [[]], [ [], [[]] ] ] - let mut stream = RlpStream::new_list(3); - stream.append_list(0); - stream.append_list(1).append_list(0); - stream.append_list(2).append_list(0).append_list(1).append_list(0); - let _ = stream.out().unwrap(); - }); + b.iter(|| { + // [ [], [[]], [ [], [[]] ] ] + let mut stream = RlpStream::new_list(3); + stream.append_list(0); + stream.append_list(1).append_list(0); + stream.append_list(2).append_list(0).append_list(1).append_list(0); + let _ = stream.out().unwrap(); + }); } #[bench] fn bench_decode_nested_empty_lists(b: &mut Bencher) { - b.iter( || { - // [ [], [[]], [ [], [[]] ] ] - let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; - let rlp = Rlp::new(&data); - let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); - let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); - let nested_rlp = rlp.at(2).unwrap(); - let _v2a: Vec = Decodable::decode(&nested_rlp.at(0).unwrap()).unwrap(); - let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1).unwrap()).unwrap(); - }); + b.iter(|| { + // [ [], [[]], [ [], [[]] ] ] + let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; + let rlp = Rlp::new(&data); + let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); + let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); + let nested_rlp = rlp.at(2).unwrap(); + let _v2a: Vec = Decodable::decode(&nested_rlp.at(0).unwrap()).unwrap(); + let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1).unwrap()).unwrap(); + }); } #[bench] fn bench_stream_1000_empty_lists(b: &mut Bencher) { - b.iter( || { - let mut stream = RlpStream::new_list(1000); - for _ in 0..1000 { - stream.append_list(0); - } - let _ = stream.out().unwrap(); - }); + b.iter(|| { + let mut stream = RlpStream::new_list(1000); + for _ in 0..1000 { + stream.append_list(0); + } + let _ = stream.out().unwrap(); + }); } diff --git a/src/rlp.rs b/src/rlp.rs index ee74c3a33..8fbe8cf8f 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -7,38 +7,75 @@ //! //! ```rust //! extern crate ethcore_util; -//! use ethcore_util::rlp::{RlpStream}; +//! use ethcore_util::rlp::{Rlp, RlpStream, Decodable}; //! //! fn encode_value() { -//! // 1029 -//! let mut stream = RlpStream::new(); -//! stream.append(&1029u32); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0x82, 0x04, 0x05]); +//! // 1029 +//! let mut stream = RlpStream::new(); +//! stream.append(&1029u32); +//! let out = stream.out().unwrap(); +//! assert_eq!(out, vec![0x82, 0x04, 0x05]); //! } //! //! fn encode_list() { -//! // [ "cat", "dog" ] -//! let mut stream = RlpStream::new_list(2); -//! stream.append(&"cat").append(&"dog"); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); +//! // [ "cat", "dog" ] +//! let mut stream = RlpStream::new_list(2); +//! stream.append(&"cat").append(&"dog"); +//! let out = stream.out().unwrap(); +//! assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); //! } //! //! fn encode_list2() { -//! // [ [], [[]], [ [], [[]] ] ] -//! let mut stream = RlpStream::new_list(3); -//! stream.append_list(0); -//! stream.append_list(1).append_list(0); -//! stream.append_list(2).append_list(0).append_list(1).append_list(0); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); +//! // [ [], [[]], [ [], [[]] ] ] +//! let mut stream = RlpStream::new_list(3); +//! stream.append_list(0); +//! stream.append_list(1).append_list(0); +//! stream.append_list(2).append_list(0).append_list(1).append_list(0); +//! let out = stream.out().unwrap(); +//! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); +//! } +//! +//! fn decode_value() { +//! // 0x102456789abcdef +//! let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; +//! let rlp = Rlp::new(&data); +//! let _ = u64::decode(&rlp).unwrap(); +//! } +//! +//! fn decode_string() { +//! // "cat" +//! let data = vec![0x83, b'c', b'a', b't']; +//! let rlp = Rlp::new(&data); +//! let _ = String::decode(&rlp).unwrap(); +//! } +//! +//! fn decode_list() { +//! // ["cat", "dog"] +//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +//! let rlp = Rlp::new(&data); +//! let _ : Vec = Decodable::decode(&rlp).unwrap(); +//! } +//! +//! fn decode_list2() { +//! // [ [], [[]], [ [], [[]] ] ] +//! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; +//! let rlp = Rlp::new(&data); +//! let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); +//! let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); +//! let nested_rlp = rlp.at(2).unwrap(); +//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0).unwrap()).unwrap(); +//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1).unwrap()).unwrap(); //! } //! //! fn main() { -//! encode_value(); -//! encode_list(); -//! encode_list2(); +//! encode_value(); +//! encode_list(); +//! encode_list2(); +//! +//! decode_value(); +//! decode_string(); +//! decode_list(); +//! decode_list2(); //! } //! ``` //! From 90c66a8079b553cfd5cbff58e5263eee431087ea Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 11:25:00 +0100 Subject: [PATCH 062/379] rlp encoding and decoding of fixed sized hashes --- src/bytes.rs | 39 ++++++++++++++++++++++++++++++++++++++- src/hash.rs | 5 +++++ src/rlp.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/bytes.rs b/src/bytes.rs index 0f82748a1..c08c4ee2a 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -35,8 +35,10 @@ //! ``` use std::fmt; +use std::cmp::Ordering; use std::error::Error as StdError; use uint::{U128, U256}; +use hash::FixedHash; /// Vector of bytes pub type Bytes = Vec; @@ -173,10 +175,26 @@ macro_rules! impl_uint_to_bytes { impl_uint_to_bytes!(U256); impl_uint_to_bytes!(U128); +impl ToBytes for T where T: FixedHash { + fn to_bytes(&self) -> Vec { + let mut res: Vec = vec![]; + res.reserve(T::size()); + + unsafe { + use std::ptr; + ptr::copy(self.bytes().as_ptr(), res.as_mut_ptr(), T::size()); + res.set_len(T::size()); + } + + res + } +} + /// Error returned when FromBytes conversation goes wrong #[derive(Debug, PartialEq, Eq)] pub enum FromBytesError { - UnexpectedEnd + DataIsTooShort, + DataIsTooLong } impl StdError for FromBytesError { @@ -256,3 +274,22 @@ macro_rules! impl_uint_from_bytes { impl_uint_from_bytes!(U256); impl_uint_from_bytes!(U128); + +impl FromBytes for T where T: FixedHash { + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + match bytes.len().cmp(&T::size()) { + Ordering::Less => return Err(FromBytesError::DataIsTooShort), + Ordering::Greater => return Err(FromBytesError::DataIsTooLong), + Ordering::Equal => () + }; + + unsafe { + use std::{mem, ptr}; + + let mut res: T = mem::uninitialized(); + ptr::copy(bytes.as_ptr(), res.mut_bytes().as_mut_ptr(), T::size()); + + Ok(res) + } + } +} diff --git a/src/hash.rs b/src/hash.rs index 0dfd4f487..a0493f2bb 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -14,6 +14,7 @@ pub trait FixedHash: Sized + BytesConvertable { fn new() -> Self; fn random() -> Self; fn randomize(&mut self); + fn size() -> usize; fn mut_bytes(&mut self) -> &mut [u8]; fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; fn bloom_part(&self, m: usize) -> T where T: FixedHash; @@ -47,6 +48,10 @@ macro_rules! impl_hash { rng.fill_bytes(&mut self.0); } + fn size() -> usize { + $size + } + fn mut_bytes(&mut self) -> &mut [u8] { &mut self.0 } diff --git a/src/rlp.rs b/src/rlp.rs index 8fbe8cf8f..4b6df2c6e 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -748,6 +748,19 @@ mod tests { run_encode_tests(tests); } + #[test] + fn encode_address() { + use hash::*; + + let tests = vec![ + ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) + ]; + run_encode_tests(tests); + } + #[test] fn encode_vector_u8() { let tests = vec![ @@ -891,6 +904,19 @@ mod tests { run_decode_tests(tests); } + #[test] + fn decode_address() { + use hash::*; + + let tests = vec![ + DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) + ]; + run_decode_tests(tests); + } + #[test] fn decode_vector_u8() { let tests = vec![ From c55956c7b4d5a5eb3791bdbf41cac5f884709ac7 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 13:08:30 +0100 Subject: [PATCH 063/379] Filter trait --- src/filter.rs | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 50 insertions(+) create mode 100644 src/filter.rs diff --git a/src/filter.rs b/src/filter.rs new file mode 100644 index 000000000..92e953c04 --- /dev/null +++ b/src/filter.rs @@ -0,0 +1,49 @@ +//! multilevel bloom filter interface +use hash::*; +use std::collections::HashMap; + +/// Represents bloom index in cache +/// +/// On bloom level 0, all positions represent different blooms. +/// On higher levels multiple positions represent one bloom +/// and should be transformed to `BlockIndex` to get index of this bloom +#[derive(Eq, PartialEq, Hash)] +pub struct BloomIndex { + level: u8, + level_index: usize, + index: usize, +} + +pub trait FilterDataSource { + /// returns reference to log at given position if it exists + fn bloom_at_index(&self, index: &BloomIndex) -> Option<&H2048>; +} + +pub trait Filter: Sized { + /// creates new filter instance + fn new(data_source: &T, index_size: usize, levels: u8) -> Self where T: FilterDataSource; + + /// converts block number and level to `BloomIndex` + fn bloom_index(&self, block_number: usize, level: u8) -> BloomIndex; + + /// add new bloom to all levels + fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap; + + /// add new blooms starting from block number + fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap; + + /// reset bloom at level 0 and forces rebuild on higher levels + fn reset_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap; + + /// sets lowest level bloom to 0 and forces rebuild on higher levels + fn clear_bloom(&self, block_number: usize) -> HashMap; + + /// returns numbers of blocks that may contain Address + fn blocks_with_address(&self, address: &Address) -> Vec; + + /// returns numbers of blocks that may contain Topic + fn blocks_with_topics(&self, topic: &H256) -> Vec; + + /// returns numbers of blocks that may log bloom + fn blocks_with_bloom(&self, bloom: &H2048) -> Vec; +} diff --git a/src/lib.rs b/src/lib.rs index acb03afb5..7ebf91f51 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ pub mod sha3; pub mod hashdb; pub mod memorydb; pub mod math; +pub mod filter; //pub mod network; From f0f70f801e89252ee637fca705fb4f2276ef3b16 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 17:08:38 +0100 Subject: [PATCH 064/379] chainfilter almost done --- Cargo.toml | 1 + src/chainfilter.rs | 265 +++++++++++++++++++++++++++++++++++++++++++++ src/filter.rs | 28 ++--- src/lib.rs | 4 + src/macros.rs | 11 ++ src/math.rs | 1 - 6 files changed, 296 insertions(+), 14 deletions(-) create mode 100644 src/chainfilter.rs create mode 100644 src/macros.rs diff --git a/Cargo.toml b/Cargo.toml index 19c64113c..c4358b8c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,4 @@ mio = "0.*" rand = "0.*" tiny-keccak = "0.3" rocksdb = "0.2.1" +num = "0.1" diff --git a/src/chainfilter.rs b/src/chainfilter.rs new file mode 100644 index 000000000..d5f0873ac --- /dev/null +++ b/src/chainfilter.rs @@ -0,0 +1,265 @@ +//! basic implementation of multilevel bloom filter +use std::collections::{HashMap, HashSet}; +use hash::*; +use filter::*; +use sha3::*; +use num::pow; + +pub struct MemoryCache { + blooms: HashMap, +} + +impl MemoryCache { + pub fn new() -> MemoryCache { + MemoryCache { blooms: HashMap::new() } + } + + pub fn insert_blooms(&mut self, blooms: HashMap) { + self.blooms.extend(blooms); + } +} + +impl FilterDataSource for MemoryCache { + fn bloom_at_index(&self, index: &BloomIndex) -> Option<&H2048> { + self.blooms.get(index) + } +} + +pub struct ChainFilter<'a, D> + where D: FilterDataSource + 'a +{ + data_source: &'a D, + index_size: usize, + levels: u8, + level_sizes: HashMap, +} + +impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource +{ + /// creates new filter instance + pub fn new(data_source: &'a D, index_size: usize, levels: u8) -> Self { + let mut filter = ChainFilter { + data_source: data_source, + index_size: index_size, + levels: levels, + level_sizes: HashMap::new(), + }; + + // cache level sizes, so we do not have to calculate them all the time + for i in 0..levels { + filter.level_sizes.insert(i, pow(index_size, i as usize)); + } + + filter + } + + /// unsafely get level size + fn level_size(&self, level: u8) -> usize { + *self.level_sizes.get(&level).unwrap() + } + + /// converts block number and level to `BloomIndex` + fn bloom_index(&self, block_number: usize, level: u8) -> BloomIndex { + BloomIndex { + level: level, + index: block_number / self.level_size(level), + } + } + + /// return bloom which are dependencies for given index + fn lower_level_bloom_indexes(&self, index: &BloomIndex) -> HashSet { + let mut indexes: HashSet = HashSet::with_capacity(self.index_size); + + // this is the lower level + if index.level == 0 { + return indexes; + } + + let new_level = index.level - 1; + let offset = self.index_size * index.index; + + for i in 0..self.index_size { + indexes.insert(BloomIndex { + level: new_level, + index: offset + i + }); + } + + indexes + } +} + +impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource +{ + /// add new bloom to all levels + /// + /// BitOr new bloom with all levels of filter + fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { + let mut result: HashMap = HashMap::new(); + + for level in 0..self.levels { + let bloom_index = self.bloom_index(block_number, level); + let new_bloom = match self.data_source.bloom_at_index(&bloom_index) { + Some(old_bloom) => old_bloom | bloom, + None => bloom.clone() + }; + + result.insert(bloom_index, new_bloom); + } + + result + } + + /// add new blooms starting from block number + /// + /// BitOr new blooms with all levels of filter + fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap { + let mut result: HashMap = HashMap::new(); + + for level in 0..self.levels { + for i in 0..blooms.len() { + let bloom_index = self.bloom_index(block_number + i, level); + let is_new_bloom = match result.get_mut(&bloom_index) { + + // it was already modified + Some(to_shift) => { + *to_shift = &blooms[i] | to_shift; + false + }, + None => true + }; + + // it hasn't been modified yet + if is_new_bloom { + let new_bloom = match self.data_source.bloom_at_index(&bloom_index) { + Some(old_bloom) => old_bloom | &blooms[i], + None => blooms[i].clone() + }; + result.insert(bloom_index, new_bloom); + } + } + } + + result + } + + /// reset bloom at level 0 and forces rebuild on higher levels + fn reset_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { + let mut result: HashMap = HashMap::new(); + + let mut reset_index = self.bloom_index(block_number, 0); + result.insert(reset_index.clone(), bloom.clone()); + + for level in 1..self.levels { + let index = self.bloom_index(block_number, level); + let lower_indexes = self.lower_level_bloom_indexes(&index); + let new_bloom = lower_indexes.into_iter() + .filter(|li| li != &reset_index) + .map(|li| self.data_source.bloom_at_index(&li)) + .filter_map(|b| b) + .fold(H2048::new(), | acc, bloom | { &acc | bloom }); + + reset_index = index.clone(); + result.insert(index, &new_bloom | bloom); + } + + result + } + + /// sets lowest level bloom to 0 and forces rebuild on higher levels + fn clear_bloom(&self, block_number: usize) -> HashMap { + self.reset_bloom(&H2048::new(), block_number) + } + + /// returns numbers of blocks that may contain Address + fn blocks_with_address(&self, address: &Address, from_block: usize ,to_block: usize) -> Vec { + let mut bloom = H2048::new(); + bloom.shift_bloom(&address.sha3()); + self.blocks_with_bloom(&bloom, from_block, to_block) + } + + /// returns numbers of blocks that may contain Topic + fn blocks_with_topics(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec { + let mut bloom = H2048::new(); + bloom.shift_bloom(&topic.sha3()); + self.blocks_with_bloom(&bloom, from_block, to_block) + } + + /// returns numbers of blocks that may log bloom + fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec { + panic!(); + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use hash::*; + use filter::*; + use chainfilter::*; + + #[test] + fn test_level_size() { + let cache = MemoryCache::new(); + let filter = ChainFilter::new(&cache, 16, 3); + assert_eq!(filter.level_size(0), 1); + assert_eq!(filter.level_size(1), 16); + assert_eq!(filter.level_size(2), 256); + } + + #[test] + fn test_bloom_index() { + let cache = MemoryCache::new(); + let filter = ChainFilter::new(&cache, 16, 3); + + let bi0 = filter.bloom_index(0, 0); + assert_eq!(bi0.level, 0); + assert_eq!(bi0.index, 0); + + let bi1 = filter.bloom_index(1, 0); + assert_eq!(bi1.level, 0); + assert_eq!(bi1.index, 1); + + let bi2 = filter.bloom_index(2, 0); + assert_eq!(bi2.level, 0); + assert_eq!(bi2.index, 2); + + let bi3 = filter.bloom_index(3, 1); + assert_eq!(bi3.level, 1); + assert_eq!(bi3.index, 0); + + let bi4 = filter.bloom_index(15, 1); + assert_eq!(bi4.level, 1); + assert_eq!(bi4.index, 0); + + let bi5 = filter.bloom_index(16, 1); + assert_eq!(bi5.level, 1); + assert_eq!(bi5.index, 1); + + let bi6 = filter.bloom_index(255, 2); + assert_eq!(bi6.level, 2); + assert_eq!(bi6.index, 0); + + let bi7 = filter.bloom_index(256, 2); + assert_eq!(bi7.level, 2); + assert_eq!(bi7.index, 1); + } + + #[test] + fn test_lower_level_bloom_indexes() { + let cache = MemoryCache::new(); + let filter = ChainFilter::new(&cache, 16, 3); + + let bi = filter.bloom_index(256, 2); + assert_eq!(bi.level, 2); + assert_eq!(bi.index, 1); + + let mut ebis = HashSet::with_capacity(16); + for i in 16..32 { + ebis.insert(BloomIndex::new(1, i)); + } + + let bis = filter.lower_level_bloom_indexes(&bi); + assert_eq!(ebis, bis); + } +} diff --git a/src/filter.rs b/src/filter.rs index 92e953c04..044c59563 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -7,11 +7,19 @@ use std::collections::HashMap; /// On bloom level 0, all positions represent different blooms. /// On higher levels multiple positions represent one bloom /// and should be transformed to `BlockIndex` to get index of this bloom -#[derive(Eq, PartialEq, Hash)] +#[derive(Eq, PartialEq, Hash, Clone, Debug)] pub struct BloomIndex { - level: u8, - level_index: usize, - index: usize, + pub level: u8, + pub index: usize, +} + +impl BloomIndex { + pub fn new(level: u8, index: usize) -> BloomIndex { + BloomIndex { + level: level, + index: index, + } + } } pub trait FilterDataSource { @@ -20,12 +28,6 @@ pub trait FilterDataSource { } pub trait Filter: Sized { - /// creates new filter instance - fn new(data_source: &T, index_size: usize, levels: u8) -> Self where T: FilterDataSource; - - /// converts block number and level to `BloomIndex` - fn bloom_index(&self, block_number: usize, level: u8) -> BloomIndex; - /// add new bloom to all levels fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap; @@ -39,11 +41,11 @@ pub trait Filter: Sized { fn clear_bloom(&self, block_number: usize) -> HashMap; /// returns numbers of blocks that may contain Address - fn blocks_with_address(&self, address: &Address) -> Vec; + fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec; /// returns numbers of blocks that may contain Topic - fn blocks_with_topics(&self, topic: &H256) -> Vec; + fn blocks_with_topics(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec; /// returns numbers of blocks that may log bloom - fn blocks_with_bloom(&self, bloom: &H2048) -> Vec; + fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec; } diff --git a/src/lib.rs b/src/lib.rs index 7ebf91f51..e7802b114 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,8 +7,11 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; +extern crate num; #[macro_use] extern crate log; +#[macro_use] +pub mod macros; pub mod error; pub mod hash; @@ -22,6 +25,7 @@ pub mod hashdb; pub mod memorydb; pub mod math; pub mod filter; +pub mod chainfilter; //pub mod network; diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 000000000..69286a340 --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,11 @@ +macro_rules! map( + { $($key:expr => $value:expr),+ } => { + { + let mut m = ::std::collections::HashMap::new(); + $( + m.insert($key, $value); + )+ + m + } + }; +); diff --git a/src/math.rs b/src/math.rs index ba039f3d0..c85c4653e 100644 --- a/src/math.rs +++ b/src/math.rs @@ -7,4 +7,3 @@ pub fn log2(x: usize) -> u32 { let n = x.leading_zeros(); ::std::mem::size_of::() as u32 * 8 - n } - From e769406b92d5b6ac0d4673edcccd759508044ded Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 17:10:14 +0100 Subject: [PATCH 065/379] rustfmt --- src/chainfilter.rs | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index d5f0873ac..2b7754d3b 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -81,7 +81,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource for i in 0..self.index_size { indexes.insert(BloomIndex { level: new_level, - index: offset + i + index: offset + i, }); } @@ -96,12 +96,12 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource /// BitOr new bloom with all levels of filter fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { let mut result: HashMap = HashMap::new(); - + for level in 0..self.levels { - let bloom_index = self.bloom_index(block_number, level); + let bloom_index = self.bloom_index(block_number, level); let new_bloom = match self.data_source.bloom_at_index(&bloom_index) { Some(old_bloom) => old_bloom | bloom, - None => bloom.clone() + None => bloom.clone(), }; result.insert(bloom_index, new_bloom); @@ -122,18 +122,18 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource let is_new_bloom = match result.get_mut(&bloom_index) { // it was already modified - Some(to_shift) => { + Some(to_shift) => { *to_shift = &blooms[i] | to_shift; false - }, - None => true + } + None => true, }; // it hasn't been modified yet if is_new_bloom { let new_bloom = match self.data_source.bloom_at_index(&bloom_index) { Some(old_bloom) => old_bloom | &blooms[i], - None => blooms[i].clone() + None => blooms[i].clone(), }; result.insert(bloom_index, new_bloom); } @@ -149,16 +149,16 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource let mut reset_index = self.bloom_index(block_number, 0); result.insert(reset_index.clone(), bloom.clone()); - + for level in 1..self.levels { let index = self.bloom_index(block_number, level); let lower_indexes = self.lower_level_bloom_indexes(&index); let new_bloom = lower_indexes.into_iter() - .filter(|li| li != &reset_index) - .map(|li| self.data_source.bloom_at_index(&li)) - .filter_map(|b| b) - .fold(H2048::new(), | acc, bloom | { &acc | bloom }); - + .filter(|li| li != &reset_index) + .map(|li| self.data_source.bloom_at_index(&li)) + .filter_map(|b| b) + .fold(H2048::new(), |acc, bloom| &acc | bloom); + reset_index = index.clone(); result.insert(index, &new_bloom | bloom); } @@ -172,7 +172,11 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource } /// returns numbers of blocks that may contain Address - fn blocks_with_address(&self, address: &Address, from_block: usize ,to_block: usize) -> Vec { + fn blocks_with_address(&self, + address: &Address, + from_block: usize, + to_block: usize) + -> Vec { let mut bloom = H2048::new(); bloom.shift_bloom(&address.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) @@ -254,7 +258,7 @@ mod tests { assert_eq!(bi.level, 2); assert_eq!(bi.index, 1); - let mut ebis = HashSet::with_capacity(16); + let mut ebis = HashSet::with_capacity(16); for i in 16..32 { ebis.insert(BloomIndex::new(1, i)); } From 4640d64965aa7220c8a097a2374a7f35b109be6c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 18:29:50 +0100 Subject: [PATCH 066/379] Reference-counted rocksdb backed DB. Just need DB I/O now. --- src/error.rs | 29 ++++++++++++++++- src/memorydb.rs | 23 ++++++------- src/overlaydb.rs | 84 ++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 110 insertions(+), 26 deletions(-) diff --git a/src/error.rs b/src/error.rs index 9dc471f67..bb09e2e1b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,9 +1,17 @@ +#![feature(concat_idents)] + use rustc_serialize::hex::*; +#[derive(Debug)] +pub enum BaseDataError { + NegativelyReferencedHash, +} + #[derive(Debug)] pub enum EthcoreError { FromHex(FromHexError), - BadSize + BaseData(BaseDataError), + BadSize, } impl From for EthcoreError { @@ -11,3 +19,22 @@ impl From for EthcoreError { EthcoreError::FromHex(err) } } + +impl From for EthcoreError { + fn from(err: BaseDataError) -> EthcoreError { + EthcoreError::BaseData(err) + } +} + +// TODO: uncomment below once https://github.com/rust-lang/rust/issues/27336 sorted. +/*macro_rules! assimilate { + ($name:ident) => ( + impl From for EthcoreError { + fn from(err: concat_idents!($name, Error)) -> EthcoreError { + EthcoreError:: $name (err) + } + } + ) +} +assimilate!(FromHex); +assimilate!(BaseData);*/ \ No newline at end of file diff --git a/src/memorydb.rs b/src/memorydb.rs index 2f54b4524..cbb9e426b 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -4,6 +4,7 @@ use hash::*; use bytes::*; use sha3::*; use hashdb::*; +use std::mem; use std::collections::HashMap; #[derive(Debug,Clone)] @@ -85,19 +86,19 @@ impl MemoryDB { for empty in empties { self.data.remove(&empty); } } - /// Grab the number of references a particular `key` has. Returns None if the key - /// doesn't exist. - fn refs(&self, key: &H256) -> Option { - self.data.get(key).map(|&(_, rc)| rc) - } - - /// Grab the value associated with a particular `key`. Returns None if the key + /// Grab the raw information associated with a key. Returns None if the key /// doesn't exist. /// - /// Even when Some is returned, this is only guaranteed to return something useful - /// when `refs(key) > 0`. - fn value(&self, key: &H256) -> Option<&Bytes> { - self.data.get(key).map(|&(ref d, _)| d) + /// Even when Some is returned, the data is only guaranteed to be useful + /// when the refs > 0. + pub fn raw(&self, key: &H256) -> Option<&(Bytes, i32)> { + self.data.get(key) + } + + pub fn drain(&mut self) -> HashMap { + let mut data = HashMap::new(); + mem::swap(&mut self.data, &mut data); + data } } diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 99d66cd5f..cb6d396c4 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -1,59 +1,115 @@ //! Disk-backed HashDB implementation. +use error::*; use hash::*; use bytes::*; use sha3::*; use hashdb::*; use memorydb::*; use std::ops::*; +use std::sync::*; use rocksdb::{DB, Writable}; #[derive(Clone)] pub struct OverlayDB { overlay: MemoryDB, - backing: DB, + backing: Arc, } impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - fn new(backing: DB) { - self.backing = backing; - overlay = MemoryDB::new(); + fn new(backing: DB) -> OverlayDB { + OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) } } - /// Commit all memory operations to the backing database. - fn commit(&mut self) { - unimplemented!(); + /// Commit all memory operations to the backing database. + fn commit(&mut self) -> Result { + let mut ret = 0u32; + for i in self.overlay.drain().into_iter() { + let (key, (value, rc)) = i; + if rc != 0 { + let new_entry = match self.payload(&key) { + Some(x) => { + let (back_value, back_rc) = x; + if back_rc + rc < 0 { + return Err(From::from(BaseDataError::NegativelyReferencedHash)); + } + self.put_payload(&key, (&back_value, rc + back_rc)); + } + None => { + self.put_payload(&key, (&value, rc)); + } + }; + ret += 1; + } + } + Ok(ret) } /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { unimplemented!(); } + + /// Get the refs and value of the given key. + fn put_payload(&self, key: &H256, payload: (&Bytes, i32)) { + unimplemented!(); + } } impl HashDB for OverlayDB { fn lookup(&self, key: &H256) -> Option { - // TODO: return ok if positive; if negative, check backing - might be enough references there to make + // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. - let k = self.overlay.data.get(key); + let k = self.overlay.raw(key); match k { Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), _ => { let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); match self.payload(key) { - Some((d, rc)) if rc + memrc > 0 => Some(d), + Some(x) => { + let (d, rc) = x; + if rc + memrc > 0 { + Some(d) + } + else { + None + } + } + // Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done. + //Some((d, rc)) if rc + memrc > 0 => Some(d), _ => None, } } } } fn exists(&self, key: &H256) -> bool { - // TODO: copy and adapt code above. - m_overlay.exists(key) + // return ok if positive; if negative, check backing - might be enough references there to make + // it positive again. + let k = self.overlay.raw(key); + match k { + Some(&(ref d, rc)) if rc > 0 => true, + _ => { + let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); + match self.payload(key) { + Some(x) => { + let (d, rc) = x; + if rc + memrc > 0 { + true + } + else { + false + } + } + // Replace above match arm with this once https://github.com/rust-lang/rust/issues/15287 is done. + //Some((d, rc)) if rc + memrc > 0 => true, + _ => false, + } + } + } } - fn insert(&mut self, value: &[u8]) -> H256 { m_overlay.insert(value) } - fn kill(&mut self, key: &H256) { m_overlay.kill(key); } + fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } + fn kill(&mut self, key: &H256) { self.overlay.kill(key); } } #[test] From 7733faaab5961e947da66b37be2cdee17317b0df Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 19:03:50 +0100 Subject: [PATCH 067/379] overlaydb put_payload. --- src/overlaydb.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index cb6d396c4..c59be085a 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -4,6 +4,7 @@ use error::*; use hash::*; use bytes::*; use sha3::*; +use rlp::*; use hashdb::*; use memorydb::*; use std::ops::*; @@ -34,10 +35,10 @@ impl OverlayDB { if back_rc + rc < 0 { return Err(From::from(BaseDataError::NegativelyReferencedHash)); } - self.put_payload(&key, (&back_value, rc + back_rc)); + self.put_payload(&key, (back_value, rc + back_rc)); } None => { - self.put_payload(&key, (&value, rc)); + self.put_payload(&key, (value, rc)); } }; ret += 1; @@ -48,12 +49,20 @@ impl OverlayDB { /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { - unimplemented!(); + db.get(&key.bytes()) + .expect("Low-level database error. Some issue with your hard disk?") + .map(|d| { + Rlp r(d.deref()); + r(Bytes, i32) + }) } /// Get the refs and value of the given key. - fn put_payload(&self, key: &H256, payload: (&Bytes, i32)) { - unimplemented!(); + fn put_payload(&self, key: &H256, payload: (Bytes, i32)) { + let mut s = RlpStream::new_list(2); + s.append(payload.1); + s.append(payload.0); + backing.put(&key.bytes(), &s.out().unwrap()); } } From 3e28c2da313ad0d4ebcbcded052b796c05b89e7e Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 19:11:04 +0100 Subject: [PATCH 068/379] bloom filters --- src/chainfilter.rs | 113 +++++++++++++++++++++++++++++++++++++++++---- src/hash.rs | 7 ++- 2 files changed, 111 insertions(+), 9 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 2b7754d3b..902ee3a11 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -1,5 +1,5 @@ //! basic implementation of multilevel bloom filter -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap}; use hash::*; use filter::*; use sha3::*; @@ -14,6 +14,9 @@ impl MemoryCache { MemoryCache { blooms: HashMap::new() } } + /// inserts all blooms into cache + /// + /// TODO: verify if extend update old items pub fn insert_blooms(&mut self, blooms: HashMap) { self.blooms.extend(blooms); } @@ -67,8 +70,11 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource } /// return bloom which are dependencies for given index - fn lower_level_bloom_indexes(&self, index: &BloomIndex) -> HashSet { - let mut indexes: HashSet = HashSet::with_capacity(self.index_size); + /// + /// bloom indexes are ordered from lowest to highest + fn lower_level_bloom_indexes(&self, index: &BloomIndex) -> Vec { + //let mut indexes: HashSet = HashSet::with_capacity(self.index_size); + let mut indexes: Vec = vec![]; // this is the lower level if index.level == 0 { @@ -79,7 +85,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource let offset = self.index_size * index.index; for i in 0..self.index_size { - indexes.insert(BloomIndex { + indexes.push(BloomIndex { level: new_level, index: offset + i, }); @@ -87,6 +93,51 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource indexes } + + /// returns max filter level + fn max_level(&self) -> u8 { + self.levels - 1 + } + + /// internal function which actually does bloom search + /// TODO: optimize it, maybe non-recursive version? + /// TODO2: clean up? + fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Vec { + let mut result = vec![]; + let index = self.bloom_index(offset, level); + + match self.data_source.bloom_at_index(&index) { + None => (), + Some(level_bloom) => match level { + 0 => { + // to_block exclusive + if offset < to_block { + result.push(offset); + } + }, + _ => match level_bloom.contains(bloom) { + false => (), + true => { + let level_size = self.level_size(level - 1); + let from_index = self.bloom_index(from_block, level - 1); + let to_index = self.bloom_index(to_block, level - 1); + let res: Vec = self.lower_level_bloom_indexes(&index).into_iter() + // chose only blooms in range + .filter(|li| li.index >= from_index.index && li.index <= to_index.index) + // map them to offsets + .map(|li| li.index * level_size) + // get all blocks that may contain our bloom + .map(|off| self.blocks(bloom, from_block, to_block, level - 1, off)) + // flatten nested structure + .flat_map(|v| v) + .collect(); + return res + } + } + } + } + result + } } impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource @@ -191,16 +242,33 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource /// returns numbers of blocks that may log bloom fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec { - panic!(); + let mut result = vec![]; + // lets start from highest level + let max_level = self.max_level(); + let level_size = self.level_size(max_level); + let from_index = self.bloom_index(from_block, max_level); + let to_index = self.bloom_index(to_block, max_level); + + for index in from_index.index..to_index.index + 1 { + // offset will be used to calculate where we are right now + let offset = level_size * index; + + // go doooown! + result.extend(self.blocks(bloom, from_block, to_block, max_level, offset)); + } + + result } } #[cfg(test)] mod tests { - use std::collections::{HashMap, HashSet}; + use std::collections::{HashMap}; use hash::*; use filter::*; use chainfilter::*; + use sha3::*; + use std::str::FromStr; #[test] fn test_level_size() { @@ -258,12 +326,41 @@ mod tests { assert_eq!(bi.level, 2); assert_eq!(bi.index, 1); - let mut ebis = HashSet::with_capacity(16); + let mut ebis = vec![]; for i in 16..32 { - ebis.insert(BloomIndex::new(1, i)); + ebis.push(BloomIndex::new(1, i)); } let bis = filter.lower_level_bloom_indexes(&bi); assert_eq!(ebis, bis); } + + #[test] + fn test_basic_search() { + let index_size = 16; + let bloom_levels = 3; + + let mut cache = MemoryCache::new(); + let topic = H256::from_str("8d936b1bd3fc635710969ccfba471fb17d598d9d1971b538dd712e1e4b4f4dba").unwrap(); + + let modified_blooms = { + let filter = ChainFilter::new(&cache, index_size, bloom_levels); + let block_number = 23; + let mut bloom = H2048::new(); + bloom.shift_bloom(&topic.sha3()); + filter.add_bloom(&bloom, block_number) + }; + + // number of modified blooms should always be equal number of levels + assert_eq!(modified_blooms.len(), bloom_levels as usize); + cache.insert_blooms(modified_blooms); + + { + let filter = ChainFilter::new(&cache, index_size, bloom_levels); + let blocks = filter.blocks_with_topics(&topic, 0, 100); + println!("{:?}", blocks); + assert!(false); + } + + } } diff --git a/src/hash.rs b/src/hash.rs index a0493f2bb..c79234300 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -19,6 +19,7 @@ pub trait FixedHash: Sized + BytesConvertable { fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; fn bloom_part(&self, m: usize) -> T where T: FixedHash; fn contains_bloom(&self, b: &T) -> bool where T: FixedHash; + fn contains<'a>(&'a self, b: &'a Self) -> bool; } macro_rules! impl_hash { @@ -108,7 +109,11 @@ macro_rules! impl_hash { fn contains_bloom(&self, b: &T) -> bool where T: FixedHash { let bp: Self = b.bloom_part($size); - (&bp & self) == bp + self.contains(&bp) + } + + fn contains<'a>(&'a self, b: &'a Self) -> bool { + &(b & self) == b } } From 358962d2ea5f17a1e8f89adaad07501b760d6b1a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 19:28:59 +0100 Subject: [PATCH 069/379] Implemented, in principle. --- src/error.rs | 5 ++--- src/lib.rs | 1 - src/overlaydb.rs | 41 ++++++++++++++++++++++------------------- src/rlp.rs | 8 ++++---- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/error.rs b/src/error.rs index bb09e2e1b..f6c64a54f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,5 +1,3 @@ -#![feature(concat_idents)] - use rustc_serialize::hex::*; #[derive(Debug)] @@ -27,7 +25,8 @@ impl From for EthcoreError { } // TODO: uncomment below once https://github.com/rust-lang/rust/issues/27336 sorted. -/*macro_rules! assimilate { +/*#![feature(concat_idents)] +macro_rules! assimilate { ($name:ident) => ( impl From for EthcoreError { fn from(err: concat_idents!($name, Error)) -> EthcoreError { diff --git a/src/lib.rs b/src/lib.rs index 1c8ad9ccc..e918c7ca9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,6 @@ pub mod sha3; pub mod hashdb; pub mod memorydb; pub mod overlaydb; -pub mod bloom; pub mod math; //pub mod network; diff --git a/src/overlaydb.rs b/src/overlaydb.rs index c59be085a..d2f838bb7 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -3,7 +3,6 @@ use error::*; use hash::*; use bytes::*; -use sha3::*; use rlp::*; use hashdb::*; use memorydb::*; @@ -19,26 +18,30 @@ pub struct OverlayDB { impl OverlayDB { /// Create a new instance of OverlayDB given a `backing` database. - fn new(backing: DB) -> OverlayDB { + pub fn new(backing: DB) -> OverlayDB { OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) } } /// Commit all memory operations to the backing database. - fn commit(&mut self) -> Result { + pub fn commit(&mut self) -> Result { let mut ret = 0u32; for i in self.overlay.drain().into_iter() { let (key, (value, rc)) = i; if rc != 0 { - let new_entry = match self.payload(&key) { + match self.payload(&key) { Some(x) => { let (back_value, back_rc) = x; - if back_rc + rc < 0 { + let total_rc: i32 = back_rc as i32 + rc; + if total_rc < 0 { return Err(From::from(BaseDataError::NegativelyReferencedHash)); } - self.put_payload(&key, (back_value, rc + back_rc)); + self.put_payload(&key, (back_value, total_rc as u32)); } None => { - self.put_payload(&key, (value, rc)); + if rc < 0 { + return Err(From::from(BaseDataError::NegativelyReferencedHash)); + } + self.put_payload(&key, (value, rc as u32)); } }; ret += 1; @@ -48,21 +51,21 @@ impl OverlayDB { } /// Get the refs and value of the given key. - fn payload(&self, key: &H256) -> Option<(Bytes, i32)> { - db.get(&key.bytes()) + fn payload(&self, key: &H256) -> Option<(Bytes, u32)> { + self.backing.get(&key.bytes()) .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { - Rlp r(d.deref()); - r(Bytes, i32) + let r = Rlp::new(d.deref()); + (Bytes::decode(&r.at(1).unwrap()).unwrap(), u32::decode(&r.at(0).unwrap()).unwrap()) }) } /// Get the refs and value of the given key. - fn put_payload(&self, key: &H256, payload: (Bytes, i32)) { + fn put_payload(&self, key: &H256, payload: (Bytes, u32)) { let mut s = RlpStream::new_list(2); - s.append(payload.1); - s.append(payload.0); - backing.put(&key.bytes(), &s.out().unwrap()); + s.append(&payload.1); + s.append(&payload.0); + self.backing.put(&key.bytes(), &s.out().unwrap()).expect("Low-level database error. Some issue with your hard disk?"); } } @@ -78,7 +81,7 @@ impl HashDB for OverlayDB { match self.payload(key) { Some(x) => { let (d, rc) = x; - if rc + memrc > 0 { + if rc as i32 + memrc > 0 { Some(d) } else { @@ -97,13 +100,13 @@ impl HashDB for OverlayDB { // it positive again. let k = self.overlay.raw(key); match k { - Some(&(ref d, rc)) if rc > 0 => true, + Some(&(_, rc)) if rc > 0 => true, _ => { let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); match self.payload(key) { Some(x) => { - let (d, rc) = x; - if rc + memrc > 0 { + let (_, rc) = x; + if rc as i32 + memrc > 0 { true } else { diff --git a/src/rlp.rs b/src/rlp.rs index 4b6df2c6e..53860579b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -50,10 +50,10 @@ //! } //! //! fn decode_list() { -//! // ["cat", "dog"] -//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -//! let rlp = Rlp::new(&data); -//! let _ : Vec = Decodable::decode(&rlp).unwrap(); +//! // ["cat", "dog"] +//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +//! let rlp = Rlp::new(&data); +//! let _ : Vec = Decodable::decode(&rlp).unwrap(); //! } //! //! fn decode_list2() { From faf174c245679ae797de8172c63b02d38022fd98 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 19:48:41 +0100 Subject: [PATCH 070/379] just a few comments --- src/chainfilter.rs | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 902ee3a11..0ab56e8ea 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -5,6 +5,7 @@ use filter::*; use sha3::*; use num::pow; +/// in memory cache for blooms pub struct MemoryCache { blooms: HashMap, } @@ -16,7 +17,7 @@ impl MemoryCache { /// inserts all blooms into cache /// - /// TODO: verify if extend update old items + /// if bloom at given index already exists, overwrites it pub fn insert_blooms(&mut self, blooms: HashMap) { self.blooms.extend(blooms); } @@ -28,6 +29,7 @@ impl FilterDataSource for MemoryCache { } } +/// Should be used to find blocks in FilterDataSource pub struct ChainFilter<'a, D> where D: FilterDataSource + 'a { @@ -101,7 +103,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// internal function which actually does bloom search /// TODO: optimize it, maybe non-recursive version? - /// TODO2: clean up? + /// TODO: clean up? fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Vec { let mut result = vec![]; let index = self.bloom_index(offset, level); @@ -263,7 +265,6 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource #[cfg(test)] mod tests { - use std::collections::{HashMap}; use hash::*; use filter::*; use chainfilter::*; @@ -336,7 +337,7 @@ mod tests { } #[test] - fn test_basic_search() { + fn test_topic_basic_search() { let index_size = 16; let bloom_levels = 3; @@ -358,9 +359,27 @@ mod tests { { let filter = ChainFilter::new(&cache, index_size, bloom_levels); let blocks = filter.blocks_with_topics(&topic, 0, 100); - println!("{:?}", blocks); - assert!(false); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0], 23); } + { + let filter = ChainFilter::new(&cache, index_size, bloom_levels); + let blocks = filter.blocks_with_topics(&topic, 0, 23); + assert_eq!(blocks.len(), 0); + } + + { + let filter = ChainFilter::new(&cache, index_size, bloom_levels); + let blocks = filter.blocks_with_topics(&topic, 23, 24); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0], 23); + } + + { + let filter = ChainFilter::new(&cache, index_size, bloom_levels); + let blocks = filter.blocks_with_topics(&topic, 24, 100); + assert_eq!(blocks.len(), 0); + } } } From addd1e5ffd7426d8231d59e322de27306ed800a0 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 20:00:14 +0100 Subject: [PATCH 071/379] bloom filter cleanup --- src/chainfilter.rs | 55 ++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 0ab56e8ea..494037149 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -105,40 +105,37 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// TODO: optimize it, maybe non-recursive version? /// TODO: clean up? fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Vec { - let mut result = vec![]; let index = self.bloom_index(offset, level); - match self.data_source.bloom_at_index(&index) { - None => (), + let contains = match self.data_source.bloom_at_index(&index) { + None => false, Some(level_bloom) => match level { - 0 => { - // to_block exclusive - if offset < to_block { - result.push(offset); - } - }, - _ => match level_bloom.contains(bloom) { - false => (), - true => { - let level_size = self.level_size(level - 1); - let from_index = self.bloom_index(from_block, level - 1); - let to_index = self.bloom_index(to_block, level - 1); - let res: Vec = self.lower_level_bloom_indexes(&index).into_iter() - // chose only blooms in range - .filter(|li| li.index >= from_index.index && li.index <= to_index.index) - // map them to offsets - .map(|li| li.index * level_size) - // get all blocks that may contain our bloom - .map(|off| self.blocks(bloom, from_block, to_block, level - 1, off)) - // flatten nested structure - .flat_map(|v| v) - .collect(); - return res - } - } + // if we are on the lowest level + // take the value, exclude to_block + 0 if offset < to_block => return vec![offset], + 0 => false, + _ => level_bloom.contains(bloom) } + }; + + if contains { + let level_size = self.level_size(level - 1); + let from_index = self.bloom_index(from_block, level - 1); + let to_index = self.bloom_index(to_block, level - 1); + let res: Vec = self.lower_level_bloom_indexes(&index).into_iter() + // chose only blooms in range + .filter(|li| li.index >= from_index.index && li.index <= to_index.index) + // map them to offsets + .map(|li| li.index * level_size) + // get all blocks that may contain our bloom + .map(|off| self.blocks(bloom, from_block, to_block, level - 1, off)) + // flatten nested structure + .flat_map(|v| v) + .collect(); + return res } - result + + return vec![]; } } From 1a30d918537f8861987beb3f9705e30e2c736a32 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 20:06:28 +0100 Subject: [PATCH 072/379] removed unused line --- src/chainfilter.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 494037149..05060023b 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -75,7 +75,6 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// /// bloom indexes are ordered from lowest to highest fn lower_level_bloom_indexes(&self, index: &BloomIndex) -> Vec { - //let mut indexes: HashSet = HashSet::with_capacity(self.index_size); let mut indexes: Vec = vec![]; // this is the lower level From d1223b3b1555ca6701e9fa1342c18f02ca0b93ca Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 20:32:46 +0100 Subject: [PATCH 073/379] bloom filter uses vector instead of hashmap, added few comments --- src/chainfilter.rs | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 05060023b..9bab944c1 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -35,24 +35,26 @@ pub struct ChainFilter<'a, D> { data_source: &'a D, index_size: usize, - levels: u8, - level_sizes: HashMap, + level_sizes: Vec, } impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource { /// creates new filter instance pub fn new(data_source: &'a D, index_size: usize, levels: u8) -> Self { + if levels == 0 { + panic!("ChainFilter requires and least 1 level"); + } + let mut filter = ChainFilter { data_source: data_source, index_size: index_size, - levels: levels, - level_sizes: HashMap::new(), + level_sizes: vec![] }; // cache level sizes, so we do not have to calculate them all the time for i in 0..levels { - filter.level_sizes.insert(i, pow(index_size, i as usize)); + filter.level_sizes.push(pow(index_size, i as usize)); } filter @@ -60,7 +62,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// unsafely get level size fn level_size(&self, level: u8) -> usize { - *self.level_sizes.get(&level).unwrap() + self.level_sizes[level as usize] } /// converts block number and level to `BloomIndex` @@ -95,9 +97,14 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource indexes } + /// return number of levels + fn levels(&self) -> u8 { + self.level_sizes.len() as u8 + } + /// returns max filter level fn max_level(&self) -> u8 { - self.levels - 1 + self.level_sizes.len() as u8 - 1 } /// internal function which actually does bloom search @@ -146,7 +153,7 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { let mut result: HashMap = HashMap::new(); - for level in 0..self.levels { + for level in 0..self.levels() { let bloom_index = self.bloom_index(block_number, level); let new_bloom = match self.data_source.bloom_at_index(&bloom_index) { Some(old_bloom) => old_bloom | bloom, @@ -165,7 +172,7 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap { let mut result: HashMap = HashMap::new(); - for level in 0..self.levels { + for level in 0..self.levels() { for i in 0..blooms.len() { let bloom_index = self.bloom_index(block_number + i, level); let is_new_bloom = match result.get_mut(&bloom_index) { @@ -199,13 +206,18 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource let mut reset_index = self.bloom_index(block_number, 0); result.insert(reset_index.clone(), bloom.clone()); - for level in 1..self.levels { + for level in 1..self.levels() { let index = self.bloom_index(block_number, level); + // get all bloom indexes that were used to construct this bloom let lower_indexes = self.lower_level_bloom_indexes(&index); let new_bloom = lower_indexes.into_iter() + // skip reseted one .filter(|li| li != &reset_index) + // get blooms for these indexes .map(|li| self.data_source.bloom_at_index(&li)) + // filter existing ones .filter_map(|b| b) + // BitOr all of them .fold(H2048::new(), |acc, bloom| &acc | bloom); reset_index = index.clone(); From 9b02a8bd5f06cd5032009fc23bd3480dd31c66eb Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 20:44:38 +0100 Subject: [PATCH 074/379] cleanedup chain filter --- src/chainfilter.rs | 55 +++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 9bab944c1..6f917e060 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -110,38 +110,40 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// internal function which actually does bloom search /// TODO: optimize it, maybe non-recursive version? /// TODO: clean up? - fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Vec { + fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Option> { let index = self.bloom_index(offset, level); - let contains = match self.data_source.bloom_at_index(&index) { - None => false, + match self.data_source.bloom_at_index(&index) { + None => return None, Some(level_bloom) => match level { // if we are on the lowest level // take the value, exclude to_block - 0 if offset < to_block => return vec![offset], - 0 => false, - _ => level_bloom.contains(bloom) + 0 if offset < to_block => return Some(vec![offset]), + // return None if it is is equal to to_block + 0 => return None, + // return None if current level doesnt contain given bloom + _ if !level_bloom.contains(bloom) => return None, + // continue processing && go down + _ => () } }; - if contains { - let level_size = self.level_size(level - 1); - let from_index = self.bloom_index(from_block, level - 1); - let to_index = self.bloom_index(to_block, level - 1); - let res: Vec = self.lower_level_bloom_indexes(&index).into_iter() - // chose only blooms in range - .filter(|li| li.index >= from_index.index && li.index <= to_index.index) - // map them to offsets - .map(|li| li.index * level_size) - // get all blocks that may contain our bloom - .map(|off| self.blocks(bloom, from_block, to_block, level - 1, off)) - // flatten nested structure - .flat_map(|v| v) - .collect(); - return res - } - - return vec![]; + let level_size = self.level_size(level - 1); + let from_index = self.bloom_index(from_block, level - 1); + let to_index = self.bloom_index(to_block, level - 1); + let res: Vec = self.lower_level_bloom_indexes(&index).into_iter() + // chose only blooms in range + .filter(|li| li.index >= from_index.index && li.index <= to_index.index) + // map them to offsets + .map(|li| li.index * level_size) + // get all blocks that may contain our bloom + .map(|off| self.blocks(bloom, from_block, to_block, level - 1, off)) + // filter existing ones + .filter_map(|x| x) + // flatten nested structure + .flat_map(|v| v) + .collect(); + Some(res) } } @@ -264,7 +266,10 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource let offset = level_size * index; // go doooown! - result.extend(self.blocks(bloom, from_block, to_block, max_level, offset)); + match self.blocks(bloom, from_block, to_block, max_level, offset) { + Some(blocks) => result.extend(blocks), + None => () + }; } result From a6240c0d30346f5d99d61d67910ce4e2d504ba56 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 21:03:31 +0100 Subject: [PATCH 075/379] fixed indention issues --- src/chainfilter.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 6f917e060..01c2b8d82 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -214,13 +214,13 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource let lower_indexes = self.lower_level_bloom_indexes(&index); let new_bloom = lower_indexes.into_iter() // skip reseted one - .filter(|li| li != &reset_index) + .filter(|li| li != &reset_index) // get blooms for these indexes - .map(|li| self.data_source.bloom_at_index(&li)) + .map(|li| self.data_source.bloom_at_index(&li)) // filter existing ones - .filter_map(|b| b) + .filter_map(|b| b) // BitOr all of them - .fold(H2048::new(), |acc, bloom| &acc | bloom); + .fold(H2048::new(), |acc, bloom| &acc | bloom); reset_index = index.clone(); result.insert(index, &new_bloom | bloom); @@ -235,11 +235,7 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource } /// returns numbers of blocks that may contain Address - fn blocks_with_address(&self, - address: &Address, - from_block: usize, - to_block: usize) - -> Vec { + fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec { let mut bloom = H2048::new(); bloom.shift_bloom(&address.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) From b405c061a1bf5ca26c22ce710684f5f9bb575d58 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 21:11:10 +0100 Subject: [PATCH 076/379] lower_level_bloom_indexes use collect instead of for loop --- src/chainfilter.rs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 01c2b8d82..3ca72e4bc 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -77,24 +77,15 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// /// bloom indexes are ordered from lowest to highest fn lower_level_bloom_indexes(&self, index: &BloomIndex) -> Vec { - let mut indexes: Vec = vec![]; - - // this is the lower level + // this is the lowest level if index.level == 0 { - return indexes; + return vec![]; } let new_level = index.level - 1; let offset = self.index_size * index.index; - for i in 0..self.index_size { - indexes.push(BloomIndex { - level: new_level, - index: offset + i, - }); - } - - indexes + (0..self.index_size).map(|i| BloomIndex::new(new_level, offset + i)).collect() } /// return number of levels @@ -140,7 +131,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource .map(|off| self.blocks(bloom, from_block, to_block, level - 1, off)) // filter existing ones .filter_map(|x| x) - // flatten nested structure + // flatten nested structures .flat_map(|v| v) .collect(); Some(res) From da60046e3d862fa4019b12473d540f5ff40001bf Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 21:42:50 +0100 Subject: [PATCH 077/379] bloom filters --- src/chainfilter.rs | 97 ++++++++++++++++++++++++++++------------------ src/filter.rs | 51 ------------------------ src/lib.rs | 2 +- 3 files changed, 60 insertions(+), 90 deletions(-) delete mode 100644 src/filter.rs diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 3ca72e4bc..bfe2a5179 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -1,16 +1,45 @@ -//! basic implementation of multilevel bloom filter +//! Multilevel blockchain bloom filter. use std::collections::{HashMap}; use hash::*; -use filter::*; use sha3::*; use num::pow; -/// in memory cache for blooms +/// Represents bloom index in cache +/// +/// On cache level 0, every block bloom is represented by different index. +/// On higher cache levels, multiple block blooms are represented by one +/// index. Their `BloomIndex` can be created from block number and given level. +#[derive(Eq, PartialEq, Hash, Clone, Debug)] +pub struct BloomIndex { + pub level: u8, + pub index: usize, +} + +impl BloomIndex { + /// Default constructor for `BloomIndex` + pub fn new(level: u8, index: usize) -> BloomIndex { + BloomIndex { + level: level, + index: index, + } + } +} + +/// Types implementing this trait should provide read access for bloom filters database. +pub trait FilterDataSource { + /// returns reference to log at given position if it exists + fn bloom_at_index(&self, index: &BloomIndex) -> Option<&H2048>; +} + +/// In memory cache for blooms. +/// +/// Stores all blooms in HashMap, which indexes them by `BloomIndex`. pub struct MemoryCache { blooms: HashMap, } impl MemoryCache { + /// Default constructor for MemoryCache pub fn new() -> MemoryCache { MemoryCache { blooms: HashMap::new() } } @@ -29,7 +58,7 @@ impl FilterDataSource for MemoryCache { } } -/// Should be used to find blocks in FilterDataSource +/// Should be used for search operations on blockchain. pub struct ChainFilter<'a, D> where D: FilterDataSource + 'a { @@ -40,7 +69,9 @@ pub struct ChainFilter<'a, D> impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource { - /// creates new filter instance + /// Creates new filter instance. + /// + /// Borrows `FilterDataSource` for reading. pub fn new(data_source: &'a D, index_size: usize, levels: u8) -> Self { if levels == 0 { panic!("ChainFilter requires and least 1 level"); @@ -98,9 +129,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource self.level_sizes.len() as u8 - 1 } - /// internal function which actually does bloom search - /// TODO: optimize it, maybe non-recursive version? - /// TODO: clean up? + /// internal function which does bloom search recursively fn blocks(&self, bloom: &H2048, from_block: usize, to_block: usize, level: u8, offset: usize) -> Option> { let index = self.bloom_index(offset, level); @@ -136,14 +165,9 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource .collect(); Some(res) } -} -impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource -{ - /// add new bloom to all levels - /// - /// BitOr new bloom with all levels of filter - fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { + /// Adds new bloom to all filter levels + pub fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { let mut result: HashMap = HashMap::new(); for level in 0..self.levels() { @@ -159,10 +183,8 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource result } - /// add new blooms starting from block number - /// - /// BitOr new blooms with all levels of filter - fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap { + /// Adds new blooms starting from block number. + pub fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap { let mut result: HashMap = HashMap::new(); for level in 0..self.levels() { @@ -192,8 +214,8 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource result } - /// reset bloom at level 0 and forces rebuild on higher levels - fn reset_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { + /// Resets bloom at level 0 and forces rebuild on higher levels. + pub fn reset_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap { let mut result: HashMap = HashMap::new(); let mut reset_index = self.bloom_index(block_number, 0); @@ -204,14 +226,14 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource // get all bloom indexes that were used to construct this bloom let lower_indexes = self.lower_level_bloom_indexes(&index); let new_bloom = lower_indexes.into_iter() - // skip reseted one - .filter(|li| li != &reset_index) - // get blooms for these indexes - .map(|li| self.data_source.bloom_at_index(&li)) - // filter existing ones - .filter_map(|b| b) - // BitOr all of them - .fold(H2048::new(), |acc, bloom| &acc | bloom); + // skip reseted one + .filter(|li| li != &reset_index) + // get blooms for these indexes + .map(|li| self.data_source.bloom_at_index(&li)) + // filter existing ones + .filter_map(|b| b) + // BitOr all of them + .fold(H2048::new(), |acc, bloom| &acc | bloom); reset_index = index.clone(); result.insert(index, &new_bloom | bloom); @@ -220,27 +242,27 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource result } - /// sets lowest level bloom to 0 and forces rebuild on higher levels - fn clear_bloom(&self, block_number: usize) -> HashMap { + /// Sets lowest level bloom to 0 and forces rebuild on higher levels. + pub fn clear_bloom(&self, block_number: usize) -> HashMap { self.reset_bloom(&H2048::new(), block_number) } - /// returns numbers of blocks that may contain Address - fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec { + /// Returns numbers of blocks that may contain Address. + pub fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec { let mut bloom = H2048::new(); bloom.shift_bloom(&address.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) } - /// returns numbers of blocks that may contain Topic - fn blocks_with_topics(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec { + /// Returns numbers of blocks that may contain Topic. + pub fn blocks_with_topics(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec { let mut bloom = H2048::new(); bloom.shift_bloom(&topic.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) } - /// returns numbers of blocks that may log bloom - fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec { + /// Returns numbers of blocks that may log bloom. + pub fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec { let mut result = vec![]; // lets start from highest level let max_level = self.max_level(); @@ -266,7 +288,6 @@ impl<'a, D> Filter for ChainFilter<'a, D> where D: FilterDataSource #[cfg(test)] mod tests { use hash::*; - use filter::*; use chainfilter::*; use sha3::*; use std::str::FromStr; diff --git a/src/filter.rs b/src/filter.rs deleted file mode 100644 index 044c59563..000000000 --- a/src/filter.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! multilevel bloom filter interface -use hash::*; -use std::collections::HashMap; - -/// Represents bloom index in cache -/// -/// On bloom level 0, all positions represent different blooms. -/// On higher levels multiple positions represent one bloom -/// and should be transformed to `BlockIndex` to get index of this bloom -#[derive(Eq, PartialEq, Hash, Clone, Debug)] -pub struct BloomIndex { - pub level: u8, - pub index: usize, -} - -impl BloomIndex { - pub fn new(level: u8, index: usize) -> BloomIndex { - BloomIndex { - level: level, - index: index, - } - } -} - -pub trait FilterDataSource { - /// returns reference to log at given position if it exists - fn bloom_at_index(&self, index: &BloomIndex) -> Option<&H2048>; -} - -pub trait Filter: Sized { - /// add new bloom to all levels - fn add_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap; - - /// add new blooms starting from block number - fn add_blooms(&self, blooms: &[H2048], block_number: usize) -> HashMap; - - /// reset bloom at level 0 and forces rebuild on higher levels - fn reset_bloom(&self, bloom: &H2048, block_number: usize) -> HashMap; - - /// sets lowest level bloom to 0 and forces rebuild on higher levels - fn clear_bloom(&self, block_number: usize) -> HashMap; - - /// returns numbers of blocks that may contain Address - fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec; - - /// returns numbers of blocks that may contain Topic - fn blocks_with_topics(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec; - - /// returns numbers of blocks that may log bloom - fn blocks_with_bloom(&self, bloom: &H2048, from_block: usize, to_block: usize) -> Vec; -} diff --git a/src/lib.rs b/src/lib.rs index e7802b114..7823c7711 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,7 @@ pub mod sha3; pub mod hashdb; pub mod memorydb; pub mod math; -pub mod filter; +//pub mod filter; pub mod chainfilter; //pub mod network; From 52ccd7b094942ca7d5e23723c59db5a0faa7b0e4 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 21:54:41 +0100 Subject: [PATCH 078/379] fixed typo --- src/chainfilter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index bfe2a5179..3f66173a7 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -74,7 +74,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// Borrows `FilterDataSource` for reading. pub fn new(data_source: &'a D, index_size: usize, levels: u8) -> Self { if levels == 0 { - panic!("ChainFilter requires and least 1 level"); + panic!("ChainFilter requires at least 1 level"); } let mut filter = ChainFilter { From d6d51b456bbd651519530a72375560e71adbaa8a Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 22:10:58 +0100 Subject: [PATCH 079/379] UnsafeRlp --- src/rlp.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index 4b6df2c6e..d8d8c2e25 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -151,6 +151,54 @@ impl From for DecoderError { } } +/// Unsafe wrapper for rlp decoder. +/// +/// It assumes that you know what you are doing. Doesn't bother +/// you with error handling. +pub struct UnsafeRlp<'a> { + rlp: Rlp<'a> +} + +impl<'a> From> for UnsafeRlp<'a> { + fn from(rlp: Rlp<'a>) -> UnsafeRlp<'a> { + UnsafeRlp { rlp: rlp } + } +} + +impl<'a> From> for Rlp<'a> { + fn from(unsafe_rlp: UnsafeRlp<'a>) -> Rlp<'a> { + unsafe_rlp.rlp + } +} + +impl<'a> UnsafeRlp<'a> { + /// returns new instance of `UnsafeRlp` + pub fn new(bytes: &'a [u8]) -> UnsafeRlp<'a> { + UnsafeRlp { + rlp: Rlp::new(bytes) + } + } + + pub fn at(&self, index: usize) -> UnsafeRlp<'a> { + From::from(self.rlp.at(index).unwrap()) + } + + /// returns true if rlp is a list + pub fn is_list(&self) -> bool { + self.rlp.is_list() + } + + /// returns true if rlp is a value + pub fn is_value(&self) -> bool { + self.rlp.is_value() + } + + /// returns rlp iterator + pub fn iter(&'a self) -> RlpIterator<'a> { + self.rlp.into_iter() + } +} + impl<'a> Rlp<'a> { /// returns new instance of `Rlp` pub fn new(bytes: &'a [u8]) -> Rlp<'a> { From b0c38cb6ec3206e6a6d46bb1796749ed9f768262 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 22:16:08 +0100 Subject: [PATCH 080/379] UntrastedRlp --- src/rlp.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index d8d8c2e25..48f5299ee 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -155,31 +155,31 @@ impl From for DecoderError { /// /// It assumes that you know what you are doing. Doesn't bother /// you with error handling. -pub struct UnsafeRlp<'a> { +pub struct UntrustedRlp<'a> { rlp: Rlp<'a> } -impl<'a> From> for UnsafeRlp<'a> { - fn from(rlp: Rlp<'a>) -> UnsafeRlp<'a> { - UnsafeRlp { rlp: rlp } +impl<'a> From> for UntrustedRlp<'a> { + fn from(rlp: Rlp<'a>) -> UntrustedRlp<'a> { + UntrustedRlp { rlp: rlp } } } -impl<'a> From> for Rlp<'a> { - fn from(unsafe_rlp: UnsafeRlp<'a>) -> Rlp<'a> { +impl<'a> From> for Rlp<'a> { + fn from(unsafe_rlp: UntrustedRlp<'a>) -> Rlp<'a> { unsafe_rlp.rlp } } -impl<'a> UnsafeRlp<'a> { - /// returns new instance of `UnsafeRlp` - pub fn new(bytes: &'a [u8]) -> UnsafeRlp<'a> { - UnsafeRlp { +impl<'a> UntrustedRlp<'a> { + /// returns new instance of `UntrustedRlp` + pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { + UntrustedRlp { rlp: Rlp::new(bytes) } } - pub fn at(&self, index: usize) -> UnsafeRlp<'a> { + pub fn at(&self, index: usize) -> UntrustedRlp<'a> { From::from(self.rlp.at(index).unwrap()) } From 8c0e063f6beb95c0ab225b0df45759353acfaf1e Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 22:43:16 +0100 Subject: [PATCH 081/379] docs for chainfilter --- src/chainfilter.rs | 49 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 3f66173a7..538d1883f 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -1,4 +1,43 @@ //! Multilevel blockchain bloom filter. +//! +//! ``` +//! extern crate ethcore_util as util; +//! use std::str::FromStr; +//! use util::chainfilter::*; +//! use util::sha3::*; +//! use util::hash::*; +//! +//! fn main() { +//! let (index_size, bloom_levels) = (16, 3); +//! let mut cache = MemoryCache::new(); +//! +//! let address = Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(); +//! +//! // borrow cache for reading inside the scope +//! let modified_blooms = { +//! let filter = ChainFilter::new(&cache, index_size, bloom_levels); +//! let block_number = 39; +//! let mut bloom = H2048::new(); +//! bloom.shift_bloom(&address.sha3()); +//! filter.add_bloom(&bloom, block_number) +//! }; +//! +//! // number of updated blooms is equal number of levels +//! assert_eq!(modified_blooms.len(), bloom_levels as usize); +//! +//! // lets inserts modified blooms into the cache +//! cache.insert_blooms(modified_blooms); +//! +//! // borrow cache for another reading operations +//! { +//! let filter = ChainFilter::new(&cache, index_size, bloom_levels); +//! let blocks = filter.blocks_with_address(&address, 10, 40); +//! assert_eq!(blocks.len(), 1); +//! assert_eq!(blocks[0], 39); +//! } +//! } +//! ``` +//! use std::collections::{HashMap}; use hash::*; use sha3::*; @@ -255,7 +294,7 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource } /// Returns numbers of blocks that may contain Topic. - pub fn blocks_with_topics(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec { + pub fn blocks_with_topic(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec { let mut bloom = H2048::new(); bloom.shift_bloom(&topic.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) @@ -379,27 +418,27 @@ mod tests { { let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_topics(&topic, 0, 100); + let blocks = filter.blocks_with_topic(&topic, 0, 100); assert_eq!(blocks.len(), 1); assert_eq!(blocks[0], 23); } { let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_topics(&topic, 0, 23); + let blocks = filter.blocks_with_topic(&topic, 0, 23); assert_eq!(blocks.len(), 0); } { let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_topics(&topic, 23, 24); + let blocks = filter.blocks_with_topic(&topic, 23, 24); assert_eq!(blocks.len(), 1); assert_eq!(blocks[0], 23); } { let filter = ChainFilter::new(&cache, index_size, bloom_levels); - let blocks = filter.blocks_with_topics(&topic, 24, 100); + let blocks = filter.blocks_with_topic(&topic, 24, 100); assert_eq!(blocks.len(), 0); } } From cefb1fa9cefa37f7d197ef7340614a35791a773f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 22:54:12 +0100 Subject: [PATCH 082/379] OverlayDB complete and tested. --- src/hash.rs | 7 +++ src/overlaydb.rs | 117 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index a0493f2bb..c03650be3 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -232,9 +232,15 @@ macro_rules! impl_hash { } } + impl $from { + pub fn hex(&self) -> String { + format!("{}", self) + } + } } } +impl_hash!(H32, 4); impl_hash!(H64, 8); impl_hash!(H128, 16); impl_hash!(Address, 20); @@ -255,6 +261,7 @@ mod tests { assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); assert_eq!(format!("{}", h), "0123456789abcdef"); assert_eq!(format!("{:?}", h), "0123456789abcdef"); + assert_eq!(h.hex(), "0123456789abcdef"); assert!(h == h); assert!(h != H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xee])); assert!(h != H64([0; 8])); diff --git a/src/overlaydb.rs b/src/overlaydb.rs index d2f838bb7..2f712da24 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -8,6 +8,7 @@ use hashdb::*; use memorydb::*; use std::ops::*; use std::sync::*; +use std::env; use rocksdb::{DB, Writable}; #[derive(Clone)] @@ -22,6 +23,13 @@ impl OverlayDB { OverlayDB{ overlay: MemoryDB::new(), backing: Arc::new(backing) } } + /// Create a new instance of OverlayDB with an anonymous temporary database. + pub fn new_temp() -> OverlayDB { + let mut dir = env::temp_dir(); + dir.push(H32::random().hex()); + Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) + } + /// Commit all memory operations to the backing database. pub fn commit(&mut self) -> Result { let mut ret = 0u32; @@ -50,6 +58,8 @@ impl OverlayDB { Ok(ret) } + pub fn revert(&mut self) { self.overlay.clear(); } + /// Get the refs and value of the given key. fn payload(&self, key: &H256) -> Option<(Bytes, u32)> { self.backing.get(&key.bytes()) @@ -124,15 +134,106 @@ impl HashDB for OverlayDB { fn kill(&mut self, key: &H256) { self.overlay.kill(key); } } +#[test] +fn overlaydb_overlay_insert_and_kill() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + trie.kill(&h); + assert_eq!(trie.lookup(&h), None); +} + +#[test] +fn overlaydb_backing_insert_revert() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + trie.revert(); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); +} + +#[test] +fn overlaydb_backing_kill() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + trie.commit().unwrap(); + trie.kill(&h); + assert_eq!(trie.lookup(&h), None); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&h), None); + trie.revert(); + assert_eq!(trie.lookup(&h), None); +} + +#[test] +fn overlaydb_backing_kill_revert() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + trie.commit().unwrap(); + trie.kill(&h); + assert_eq!(trie.lookup(&h), None); + trie.revert(); + assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); +} + +#[test] +fn overlaydb_negative() { + let mut trie = OverlayDB::new_temp(); + let h = trie.insert(b"hello world"); + trie.commit().unwrap(); + trie.kill(&h); + trie.kill(&h); //bad - sends us into negative refs. + assert_eq!(trie.lookup(&h), None); + assert!(trie.commit().is_err()); +} + +#[test] +fn overlaydb_complex() { + let mut trie = OverlayDB::new_temp(); + let hfoo = trie.insert(b"foo"); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + let hbar = trie.insert(b"bar"); + assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + trie.insert(b"foo"); // two refs + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + trie.kill(&hbar); // zero refs - delete + assert_eq!(trie.lookup(&hbar), None); + trie.kill(&hfoo); // one ref - keep + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.kill(&hfoo); // zero ref - would delete, but... + assert_eq!(trie.lookup(&hfoo), None); + trie.insert(b"foo"); // one ref - keep after all. + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.commit().unwrap(); + assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + trie.kill(&hfoo); // zero ref - delete + assert_eq!(trie.lookup(&hfoo), None); + trie.commit().unwrap(); // + assert_eq!(trie.lookup(&hfoo), None); +} + #[test] fn playpen() { - let mut db: DB = DB::open_default("/tmp/test").unwrap(); - db.put(b"test", b"test2"); - match db.get(b"test") { - Ok(Some(value)) => println!("Got value {:?}", value.deref()), - Ok(None) => println!("No value for that key"), - Err(e) => println!("Gah"), + use std::fs; + { + let db: DB = DB::open_default("/tmp/test").unwrap(); + db.put(b"test", b"test2").unwrap(); + match db.get(b"test") { + Ok(Some(value)) => println!("Got value {:?}", value.deref()), + Ok(None) => println!("No value for that key"), + Err(..) => println!("Gah"), + } + db.delete(b"test").unwrap(); } - db.delete(b"test"); - assert!(false); + fs::remove_dir_all("/tmp/test").unwrap(); } \ No newline at end of file From 3f5ea22a63c3c3d7c8ba2e461f28977c3d745aa1 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 23:13:05 +0100 Subject: [PATCH 083/379] first attempt to run travis --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..e7ca3f25b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: rust +after_success: | + sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev && + wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && + tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && + sudo make install && cd ../.. && + kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* From de87de3920fece6dbb6eabaa79cc7616f6a1ec58 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 23:21:14 +0100 Subject: [PATCH 084/379] added rocksdb to .travis.yml --- .travis.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index e7ca3f25b..1069b1e3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,13 @@ language: rust -after_success: | - sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev && - wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz && - tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && - sudo make install && cd ../.. && - kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* + +before_script: + - wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz + - tar xvf rocksdb-3.8.tar.gz && cd rocksdb-rocksdb-3.8 && make shared_lib + - make install + - cd .. + +after_success: + - sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev + - wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz + - tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && sudo make install && cd ../.. && + - kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* From 7671877ed6efa0783b4d56d80949e17ba3fe99fe Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 23:23:11 +0100 Subject: [PATCH 085/379] fixed typo in travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1069b1e3f..0b33900a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: rust before_script: - wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz - - tar xvf rocksdb-3.8.tar.gz && cd rocksdb-rocksdb-3.8 && make shared_lib + - tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib - make install - cd .. From 8eb9dfb387eb523577e8e95a9e7dc19ffacab2b4 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 23:30:23 +0100 Subject: [PATCH 086/379] travis installs g++4.8 --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0b33900a9..2059c3273 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,11 @@ language: rust before_script: + # g++4.8 for C++11 which is required by rocksdb + - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get install -qq --yes --force-yes g++-4.8 + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 + - wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz - tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib - make install From 054a11a5edc010737bd24198944a1eab6cd40d25 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 23:33:01 +0100 Subject: [PATCH 087/379] apt-get update for travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2059c3273..fd0116a17 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ language: rust before_script: # g++4.8 for C++11 which is required by rocksdb - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo apt-get update -y -qq - sudo apt-get install -qq --yes --force-yes g++-4.8 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 From 7402559630bead5bedea7a0c8e929079cbd96065 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 28 Nov 2015 23:46:36 +0100 Subject: [PATCH 088/379] missing sudo --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd0116a17..03320f067 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,12 @@ language: rust +rust: + - 1.4.0 + +os: + - linux + #- osx + before_script: # g++4.8 for C++11 which is required by rocksdb - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test @@ -9,7 +16,7 @@ before_script: - wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz - tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib - - make install + - sudo make install - cd .. after_success: From 13a692cc9dd13a1d3b1aa71a100492ece51c3137 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 23:49:07 +0100 Subject: [PATCH 089/379] Docuemntation udpate and tests. --- src/overlaydb.rs | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 2f712da24..335edbc9e 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -12,6 +12,14 @@ use std::env; use rocksdb::{DB, Writable}; #[derive(Clone)] +/// Implementation of the HashDB trait for a disk-backed database with a memory overlay. +/// +/// The operations `insert()` and `kill()` take place on the memory overlay; batches of +/// such operations may be flushed to the disk-backed DB with `commit()` or discarded with +/// `revert()`. +/// +/// `lookup()` and `exists()` maintain normal behaviour - all `insert()` and `kill()` +/// queries have an immediate effect in terms of these functions. pub struct OverlayDB { overlay: MemoryDB, backing: Arc, @@ -30,7 +38,33 @@ impl OverlayDB { Self::new(DB::open_default(dir.to_str().unwrap()).unwrap()) } - /// Commit all memory operations to the backing database. + /// Commit all memory operations to the backing database. + /// + /// Returns either an error or the number of items changed in the backing database. + /// + /// Will return an error if the number of `kill()`s ever exceeds the number of + /// `insert()`s for any key. This will leave the database in an undeterminate + /// state. Don't ever let it happen. + /// + /// # Example + /// ``` + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::overlaydb::*; + /// fn main() { + /// let mut m = OverlayDB::new_temp(); + /// let key = m.insert(b"foo"); // insert item. + /// assert!(m.exists(&key)); // key exists (in memory). + /// assert_eq!(m.commit().unwrap(), 1); // 1 item changed. + /// assert!(m.exists(&key)); // key still exists (in backing). + /// m.kill(&key); // delete item. + /// assert!(!m.exists(&key)); // key "doesn't exist" (though still does in backing). + /// m.kill(&key); // oh dear... more kills than inserts for the key... + /// //m.commit().unwrap(); // this commit/unwrap would cause a panic. + /// m.revert(); // revert both kills. + /// assert!(m.exists(&key)); // key now still exists. + /// } + /// ``` pub fn commit(&mut self) -> Result { let mut ret = 0u32; for i in self.overlay.drain().into_iter() { @@ -58,6 +92,8 @@ impl OverlayDB { Ok(ret) } + /// Revert all changes though `insert()` and `kill()` to this object since the + /// last `commit()`. pub fn revert(&mut self) { self.overlay.clear(); } /// Get the refs and value of the given key. From be96b2638de20d4cd3ceefb5da9b019250bbdf88 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 28 Nov 2015 23:54:58 +0100 Subject: [PATCH 090/379] Doc tweaks. --- src/overlaydb.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 335edbc9e..7db67467b 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -92,8 +92,27 @@ impl OverlayDB { Ok(ret) } - /// Revert all changes though `insert()` and `kill()` to this object since the + /// Revert all operations on this object (i.e. `insert()`s and `kill()`s) since the /// last `commit()`. + /// + /// # Example + /// ``` + /// extern crate ethcore_util; + /// use ethcore_util::hashdb::*; + /// use ethcore_util::overlaydb::*; + /// fn main() { + /// let mut m = OverlayDB::new_temp(); + /// let foo = m.insert(b"foo"); // insert foo. + /// m.commit().unwrap(); // commit - new operations begin here... + /// let bar = m.insert(b"bar"); // insert bar. + /// m.kill(&foo); // kill foo. + /// assert!(!m.exists(&foo)); // foo is gone. + /// assert!(m.exists(&bar)); // bar is here. + /// m.revert(); // revert the last two operations. + /// assert!(m.exists(&foo)); // foo is here. + /// assert!(!m.exists(&bar)); // bar is gone. + /// } + /// ``` pub fn revert(&mut self) { self.overlay.clear(); } /// Get the refs and value of the given key. From 2e2870b06c6967280b9e43a0af22bc6511a325c7 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 00:04:57 +0100 Subject: [PATCH 091/379] attempt to run osx on travis --- .travis.yml | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03320f067..5fcde4391 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,23 +4,30 @@ rust: - 1.4.0 os: - - linux - #- osx + #- linux + - osx before_script: # g++4.8 for C++11 which is required by rocksdb - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -y -qq - - sudo apt-get install -qq --yes --force-yes g++-4.8 - - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 + - if [ $TRAVIS_OS_NAME == "linux" ]; then + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update -y -qq + sudo apt-get install -qq --yes --force-yes g++-4.8 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 - - wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz - - tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib - - sudo make install - - cd .. + wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz + tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib + sudo make install + cd .. + else + brew update + brew install rocksdb + fi after_success: - - sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev - - wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz - - tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && sudo make install && cd ../.. && - - kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* + - if [ $TRAVIS_OS_NAME == "linux" ]; then + sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev + wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz + tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && sudo make install && cd ../.. && + kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* + fi From 1171a32ceca4cae1064df188841a2ee5b67af365 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 00:11:16 +0100 Subject: [PATCH 092/379] just osx on travis --- .travis.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5fcde4391..5e6744653 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,25 +9,25 @@ os: before_script: # g++4.8 for C++11 which is required by rocksdb - - if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get update -y -qq - sudo apt-get install -qq --yes --force-yes g++-4.8 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 + #- if [ $TRAVIS_OS_NAME == "linux" ]; then + #sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + #sudo apt-get update -y -qq + #sudo apt-get install -qq --yes --force-yes g++-4.8 + #sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50 - wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz - tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib - sudo make install - cd .. - else + #wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz + #tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib + #sudo make install + #cd .. + #else brew update brew install rocksdb - fi + #fi after_success: - - if [ $TRAVIS_OS_NAME == "linux" ]; then - sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev - wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz - tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && sudo make install && cd ../.. && - kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* - fi + #- if [ $TRAVIS_OS_NAME == "linux" ]; then + #sudo apt-get install libcurl4-openssl-dev libelf-dev libdw-dev + #wget https://github.com/SimonKagstrom/kcov/archive/master.tar.gz + #tar xzf master.tar.gz && mkdir kcov-master/build && cd kcov-master/build && cmake .. && make && sudo make install && cd ../.. && + #kcov --coveralls-id=$TRAVIS_JOB_ID --exclude-pattern=/.cargo target/kcov target/debug/ethcore_util-* + #fi From 9dc6fa98deff6a679c488c021c2b5bc22c92aedd Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 00:13:32 +0100 Subject: [PATCH 093/379] travis.. --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e6744653..81a39492f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,8 +20,8 @@ before_script: #sudo make install #cd .. #else - brew update - brew install rocksdb + - brew update + - brew install rocksdb #fi after_success: From 5b57629e6d9d00546a88e36bf26f154c1ee7affd Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 01:39:39 +0100 Subject: [PATCH 094/379] First little bit of trie done. --- Cargo.toml | 1 + src/lib.rs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index c4358b8c3..31932a137 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,4 @@ rand = "0.*" tiny-keccak = "0.3" rocksdb = "0.2.1" num = "0.1" +lazy_static = "0.1.*" diff --git a/src/lib.rs b/src/lib.rs index aec47b96f..0a7c14ca9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,9 @@ extern crate tiny_keccak; extern crate num; #[macro_use] extern crate log; +#[macro_use] +extern crate lazy_static; + #[macro_use] pub mod macros; @@ -27,6 +30,7 @@ pub mod overlaydb; pub mod math; //pub mod filter; pub mod chainfilter; +pub mod trie; //pub mod network; From e04907729a0e8ec319e4a7d3401556d82bfc8bb2 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 29 Nov 2015 02:11:56 +0100 Subject: [PATCH 095/379] KeyPair, EC signature --- Cargo.toml | 8 ++- src/crypto.rs | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/hash.rs | 66 +++++++++++++++++++- src/lib.rs | 9 ++- 4 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 src/crypto.rs diff --git a/Cargo.toml b/Cargo.toml index 31932a137..0a6d5776b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,9 +11,13 @@ log = "0.3" env_logger = "0.3" rustc-serialize = "0.3" arrayvec = "0.3" -mio = "0.*" -rand = "0.*" +mio = "0.4.4" +rand = "0.3.12" +time = "0.1.34" tiny-keccak = "0.3" rocksdb = "0.2.1" num = "0.1" lazy_static = "0.1.*" +ifaces = { git = "https://github.com/dlevy47/rust-interfaces.git" } +secp256k1 = "0.5.1" +rust-crypto = "0.2.34" diff --git a/src/crypto.rs b/src/crypto.rs new file mode 100644 index 000000000..0d1ae9c05 --- /dev/null +++ b/src/crypto.rs @@ -0,0 +1,167 @@ +use hash::*; +use secp256k1::Secp256k1; +use secp256k1::key; +use rand::os::OsRng; + +pub type Secret=H256; +pub type Public=H512; +pub type Signature=H520; + +#[derive(Debug)] +pub enum CryptoError { + InvalidSecret, + InvalidPublic, + InvalidSignature, + InvalidMessage, + Io(::std::io::Error), +} + +impl From<::secp256k1::Error> for CryptoError { + fn from(e: ::secp256k1::Error) -> CryptoError { + match e { + ::secp256k1::Error::InvalidMessage => CryptoError::InvalidMessage, + ::secp256k1::Error::InvalidPublicKey => CryptoError::InvalidPublic, + ::secp256k1::Error::InvalidSignature => CryptoError::InvalidSignature, + ::secp256k1::Error::InvalidSecretKey => CryptoError::InvalidSecret, + _ => panic!("Crypto error: {:?}", e), + } + } +} + +impl From<::std::io::Error> for CryptoError { + fn from(err: ::std::io::Error) -> CryptoError { + CryptoError::Io(err) + } +} + +#[derive(Debug, PartialEq, Eq)] +/// secp256k1 Key pair +/// +/// Use `create()` to create a new random key pair. +/// +/// # Example +/// ```rust +/// extern crate ethcore_util; +/// use ethcore_util::crypto::*; +/// use ethcore_util::hash::*; +/// fn main() { +/// let pair = KeyPair::create().unwrap(); +/// let message = H256::random(); +/// let signature = sign(pair.secret(), &message).unwrap(); +/// +/// assert!(verify(pair.public(), &signature, &message).unwrap()); +/// assert_eq!(recover(&signature, &message).unwrap(), *pair.public()); +/// } +/// ``` +pub struct KeyPair { + secret: Secret, + public: Public, +} + +impl KeyPair { + /// Create a pair from secret key + pub fn from_secret(secret: Secret) -> Result { + let context = Secp256k1::new(); + let s: key::SecretKey = try!(key::SecretKey::from_slice(&context, &secret)); + let pub_key = try!(key::PublicKey::from_secret_key(&context, &s)); + let serialized = pub_key.serialize_vec(&context, false); + let p: Public = Public::from_slice(&serialized[1..65]); + Ok(KeyPair { + secret: secret, + public: p, + }) + } + /// Create a new random key pair + pub fn create() -> Result { + let context = Secp256k1::new(); + let mut rng = try!(OsRng::new()); + let (sec, publ) = try!(context.generate_keypair(&mut rng)); + let serialized = publ.serialize_vec(&context, false); + let p: Public = Public::from_slice(&serialized[1..65]); + let s: Secret = unsafe { ::std::mem::transmute(sec) }; + Ok(KeyPair { + secret: s, + public: p, + }) + } + /// Returns public key + pub fn public(&self) -> &Public { + &self.public + } + /// Returns private key + pub fn secret(&self) -> &Secret { + &self.secret + } +} + +/// Recovers Public key from signed message hash. +pub fn recover(signature: &Signature, message: &H256) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let rsig = try!(RecoverableSignature::from_compact(&context, &signature[0..64], try!(RecoveryId::from_i32(signature[64] as i32)))); + let publ = try!(context.recover(&try!(Message::from_slice(&message)), &rsig)); + let serialized = publ.serialize_vec(&context, false); + let p: Public = Public::from_slice(&serialized[1..65]); + Ok(p) +} +/// Returns siganture of message hash. +pub fn sign(secret: &Secret, message: &H256) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let sec: &key::SecretKey = unsafe { ::std::mem::transmute(secret) }; + let s = try!(context.sign_recoverable(&try!(Message::from_slice(&message)), sec)); + let (rec_id, data) = s.serialize_compact(&context); + let mut signature: ::crypto::Signature = unsafe { ::std::mem::uninitialized() }; + signature.clone_from_slice(&data); + signature[64] = rec_id.to_i32() as u8; + Ok(signature) +} +/// Verify signature. +pub fn verify(public: &Public, signature: &Signature, message: &H256) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let rsig = try!(RecoverableSignature::from_compact(&context, &signature[0..64], try!(RecoveryId::from_i32(signature[64] as i32)))); + let sig = rsig.to_standard(&context); + + let mut pdata: [u8; 65] = [4u8; 65]; + let ptr = pdata[1..].as_mut_ptr(); + let src = public.as_ptr(); + unsafe { ::std::ptr::copy_nonoverlapping(src, ptr, 64) }; + let publ = try!(key::PublicKey::from_slice(&context, &pdata)); + match context.verify(&try!(Message::from_slice(&message)), &sig, &publ) { + Ok(_) => Ok(true), + Err(Error::IncorrectSignature) => Ok(false), + Err(x) => Err(>::from(x)) + } +} + +#[cfg(test)] +mod tests { + + use std::str::FromStr; + use hash::*; + use crypto::*; + + #[test] + fn test_signature() { + let pair = KeyPair::create().unwrap(); + let message = H256::random(); + let signature = sign(pair.secret(), &message).unwrap(); + + assert!(verify(pair.public(), &signature, &message).unwrap()); + assert_eq!(recover(&signature, &message).unwrap(), *pair.public()); + } + + #[test] + fn test_invalid_key() { + assert!(KeyPair::from_secret(Secret::from_str("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap()).is_err()); + assert!(KeyPair::from_secret(Secret::from_str("0000000000000000000000000000000000000000000000000000000000000000").unwrap()).is_err()); + assert!(KeyPair::from_secret(Secret::from_str("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141").unwrap()).is_err()); + } + + #[test] + fn test_key() { + let pair = KeyPair::from_secret(Secret::from_str("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2").unwrap()).unwrap(); + assert_eq!(pair.public().hex(), "101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c"); + } +} diff --git a/src/hash.rs b/src/hash.rs index 08166559a..ad918ef3d 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,7 +1,8 @@ use std::str::FromStr; use std::fmt; +use std::ops; use std::hash::{Hash, Hasher}; -use std::ops::{Index, IndexMut, BitOr, BitAnd}; +use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitAnd}; use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; @@ -16,6 +17,8 @@ pub trait FixedHash: Sized + BytesConvertable { fn randomize(&mut self); fn size() -> usize; fn mut_bytes(&mut self) -> &mut [u8]; + fn from_slice(src: &[u8]) -> Self; + fn clone_from_slice(&mut self, src: &[u8]) -> usize; fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; fn bloom_part(&self, m: usize) -> T where T: FixedHash; fn contains_bloom(&self, b: &T) -> bool where T: FixedHash; @@ -25,7 +28,7 @@ pub trait FixedHash: Sized + BytesConvertable { macro_rules! impl_hash { ($from: ident, $size: expr) => { #[derive(Eq)] - pub struct $from ([u8; $size]); + pub struct $from (pub [u8; $size]); impl BytesConvertable for $from { fn bytes(&self) -> &[u8] { @@ -33,6 +36,24 @@ macro_rules! impl_hash { } } + impl Deref for $from { + type Target = [u8]; + #[inline] + fn deref(&self) -> &[u8] { + unsafe { + ::std::slice::from_raw_parts(self.0.as_ptr(), $size) + } + } + } + impl DerefMut for $from { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { + ::std::slice::from_raw_parts_mut(self.0.as_mut_ptr(), $size) + } + } + } + impl FixedHash for $from { fn new() -> $from { $from([0; $size]) @@ -57,6 +78,23 @@ macro_rules! impl_hash { &mut self.0 } + // TODO: remove once slice::clone_from_slice is stable + #[inline] + fn clone_from_slice(&mut self, src: &[u8]) -> usize { + let min = ::std::cmp::min($size, src.len()); + let dst = &mut self.deref_mut()[.. min]; + let src = &src[.. min]; + for i in 0..min { + dst[i] = src[i]; + } + min + } + fn from_slice(src: &[u8]) -> Self { + let mut r = Self::new(); + r.clone_from_slice(src); + r + } + fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash { let bp: Self = b.bloom_part($size); let new_self = &bp | self; @@ -186,6 +224,30 @@ macro_rules! impl_hash { &mut self.0[index] } } + impl Index> for $from { + type Output = [u8]; + + fn index<'a>(&'a self, index: ops::Range) -> &'a [u8] { + &self.0[index] + } + } + impl IndexMut> for $from { + fn index_mut<'a>(&'a mut self, index: ops::Range) -> &'a mut [u8] { + &mut self.0[index] + } + } + impl Index for $from { + type Output = [u8]; + + fn index<'a>(&'a self, _index: ops::RangeFull) -> &'a [u8] { + &self.0 + } + } + impl IndexMut for $from { + fn index_mut<'a>(&'a mut self, _index: ops::RangeFull) -> &'a mut [u8] { + &mut self.0 + } + } /// BitOr on references impl<'a> BitOr for &'a $from { diff --git a/src/lib.rs b/src/lib.rs index 0a7c14ca9..462a4035a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,8 +14,13 @@ extern crate log; extern crate lazy_static; #[macro_use] -pub mod macros; +extern crate ifaces; +extern crate time; +extern crate crypto as rcrypto; +extern crate secp256k1; +extern crate arrayvec; +pub mod macros; pub mod error; pub mod hash; pub mod uint; @@ -28,9 +33,9 @@ pub mod hashdb; pub mod memorydb; pub mod overlaydb; pub mod math; -//pub mod filter; pub mod chainfilter; pub mod trie; +pub mod crypto; //pub mod network; From f507ebf1fa1b3edfd426b8228e50ab89e9de5c38 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 09:16:15 +0100 Subject: [PATCH 096/379] rlp -> untrustedrlp in progress --- src/lib.rs | 3 +- src/overlaydb.rs | 4 +- src/rlp.rs | 147 ++++++++++++++++++++++++----------------------- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0a7c14ca9..f95f88278 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,9 +28,8 @@ pub mod hashdb; pub mod memorydb; pub mod overlaydb; pub mod math; -//pub mod filter; pub mod chainfilter; -pub mod trie; +//pub mod trie; //pub mod network; diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 7db67467b..99933d2cf 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -121,7 +121,7 @@ impl OverlayDB { .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { let r = Rlp::new(d.deref()); - (Bytes::decode(&r.at(1).unwrap()).unwrap(), u32::decode(&r.at(0).unwrap()).unwrap()) + (Bytes::decode(&r.at(1)).unwrap(), u32::decode(&r.at(0)).unwrap()) }) } @@ -291,4 +291,4 @@ fn playpen() { db.delete(b"test").unwrap(); } fs::remove_dir_all("/tmp/test").unwrap(); -} \ No newline at end of file +} diff --git a/src/rlp.rs b/src/rlp.rs index 66da7a91f..286eeac0d 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1,4 +1,4 @@ -//! Rlp serialization module +//! UntrustedRlp serialization module //! //! Types implementing `Endocable` and `Decodable` traits //! can be easily coverted to and from rlp @@ -7,7 +7,7 @@ //! //! ```rust //! extern crate ethcore_util; -//! use ethcore_util::rlp::{Rlp, RlpStream, Decodable}; +//! use ethcore_util::rlp::{UntrustedRlp, RlpStream, Decodable}; //! //! fn encode_value() { //! // 1029 @@ -38,28 +38,28 @@ //! fn decode_value() { //! // 0x102456789abcdef //! let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; -//! let rlp = Rlp::new(&data); +//! let rlp = UntrustedRlp::new(&data); //! let _ = u64::decode(&rlp).unwrap(); //! } //! //! fn decode_string() { //! // "cat" //! let data = vec![0x83, b'c', b'a', b't']; -//! let rlp = Rlp::new(&data); +//! let rlp = UntrustedRlp::new(&data); //! let _ = String::decode(&rlp).unwrap(); //! } //! //! fn decode_list() { //! // ["cat", "dog"] //! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -//! let rlp = Rlp::new(&data); +//! let rlp = UntrustedRlp::new(&data); //! let _ : Vec = Decodable::decode(&rlp).unwrap(); //! } //! //! fn decode_list2() { //! // [ [], [[]], [ [], [[]] ] ] //! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; -//! let rlp = Rlp::new(&data); +//! let rlp = UntrustedRlp::new(&data); //! let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); //! let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); //! let nested_rlp = rlp.at(2).unwrap(); @@ -89,7 +89,7 @@ use vector::InsertSlice; /// rlp container #[derive(Debug)] -pub struct Rlp<'a> { +pub struct UntrustedRlp<'a> { bytes: &'a [u8], cache: Cell, } @@ -128,10 +128,10 @@ impl ItemInfo { #[derive(Debug, PartialEq, Eq)] pub enum DecoderError { FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeValue, - BadRlp, + UntrustedRlpIsTooShort, + UntrustedRlpExpectedToBeList, + UntrustedRlpExpectedToBeValue, + BadUntrustedRlp, } impl StdError for DecoderError { fn description(&self) -> &str { @@ -155,31 +155,31 @@ impl From for DecoderError { /// /// It assumes that you know what you are doing. Doesn't bother /// you with error handling. -pub struct UntrustedRlp<'a> { - rlp: Rlp<'a> -} - -impl<'a> From> for UntrustedRlp<'a> { - fn from(rlp: Rlp<'a>) -> UntrustedRlp<'a> { - UntrustedRlp { rlp: rlp } - } +pub struct Rlp<'a> { + rlp: UntrustedRlp<'a> } impl<'a> From> for Rlp<'a> { - fn from(unsafe_rlp: UntrustedRlp<'a>) -> Rlp<'a> { + fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { + Rlp { rlp: rlp } + } +} + +impl<'a> From> for UntrustedRlp<'a> { + fn from(unsafe_rlp: Rlp<'a>) -> UntrustedRlp<'a> { unsafe_rlp.rlp } } -impl<'a> UntrustedRlp<'a> { - /// returns new instance of `UntrustedRlp` - pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { - UntrustedRlp { - rlp: Rlp::new(bytes) +impl<'a> Rlp<'a> { + /// returns new instance of `Rlp` + pub fn new(bytes: &'a [u8]) -> Rlp<'a> { + Rlp { + rlp: UntrustedRlp::new(bytes) } } - pub fn at(&self, index: usize) -> UntrustedRlp<'a> { + pub fn at(&self, index: usize) -> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } @@ -194,15 +194,15 @@ impl<'a> UntrustedRlp<'a> { } /// returns rlp iterator - pub fn iter(&'a self) -> RlpIterator<'a> { + pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { self.rlp.into_iter() } } -impl<'a> Rlp<'a> { - /// returns new instance of `Rlp` - pub fn new(bytes: &'a [u8]) -> Rlp<'a> { - Rlp { +impl<'a> UntrustedRlp<'a> { + /// returns new instance of `UntrustedRlp` + pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { + UntrustedRlp { bytes: bytes, cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), } @@ -211,28 +211,28 @@ impl<'a> Rlp<'a> { /// get container subset at given index /// /// paren container caches searched position - pub fn at(&self, index: usize) -> Result, DecoderError> { + pub fn at(&self, index: usize) -> Result, DecoderError> { if !self.is_list() { - return Err(DecoderError::RlpExpectedToBeList); + return Err(DecoderError::UntrustedRlpExpectedToBeList); } // move to cached position if it's index is less or equal to // current search index, otherwise move to beginning of list let c = self.cache.get(); let (mut bytes, to_skip) = match c.index <= index { - true => (try!(Rlp::consume(self.bytes, c.offset)), index - c.index), + true => (try!(UntrustedRlp::consume(self.bytes, c.offset)), index - c.index), false => (try!(self.consume_list_prefix()), index), }; // skip up to x items - bytes = try!(Rlp::consume_items(bytes, to_skip)); + bytes = try!(UntrustedRlp::consume_items(bytes, to_skip)); // update the cache self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); // construct new rlp - let found = try!(Rlp::item_info(bytes)); - Ok(Rlp::new(&bytes[0..found.prefix_len + found.value_len])) + let found = try!(UntrustedRlp::item_info(bytes)); + Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len])) } /// returns true if rlp is a list @@ -246,14 +246,14 @@ impl<'a> Rlp<'a> { } /// returns rlp iterator - pub fn iter(&'a self) -> RlpIterator<'a> { + pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { self.into_iter() } /// consumes first found prefix fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { - let item = try!(Rlp::item_info(self.bytes)); - let bytes = try!(Rlp::consume(self.bytes, item.prefix_len)); + let item = try!(UntrustedRlp::item_info(self.bytes)); + let bytes = try!(UntrustedRlp::consume(self.bytes, item.prefix_len)); Ok(bytes) } @@ -261,16 +261,18 @@ impl<'a> Rlp<'a> { fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { let mut result = bytes; for _ in 0..items { - let i = try!(Rlp::item_info(result)); - result = try!(Rlp::consume(result, (i.prefix_len + i.value_len))); + let i = try!(UntrustedRlp::item_info(result)); + result = try!(UntrustedRlp::consume(result, (i.prefix_len + i.value_len))); } Ok(result) } /// return first item info + /// + /// TODO: move this to decoder? fn item_info(bytes: &[u8]) -> Result { let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::RlpIsTooShort), + None => return Err(DecoderError::UntrustedRlpIsTooShort), Some(0...0x7f) => ItemInfo::new(0, 1), Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), Some(l @ 0xb8...0xbf) => { @@ -286,12 +288,12 @@ impl<'a> Rlp<'a> { let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); ItemInfo::new(prefix_len, value_len) } - _ => return Err(DecoderError::BadRlp), + _ => return Err(DecoderError::BadUntrustedRlp), }; match item.prefix_len + item.value_len <= bytes.len() { true => Ok(item), - false => Err(DecoderError::RlpIsTooShort), + false => Err(DecoderError::UntrustedRlpIsTooShort), } } @@ -299,33 +301,33 @@ impl<'a> Rlp<'a> { fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { match bytes.len() >= len { true => Ok(&bytes[len..]), - false => Err(DecoderError::RlpIsTooShort), + false => Err(DecoderError::UntrustedRlpIsTooShort), } } } /// non-consuming rlp iterator -pub struct RlpIterator<'a> { - rlp: &'a Rlp<'a>, +pub struct UntrustedRlpIterator<'a> { + rlp: &'a UntrustedRlp<'a>, index: usize, } -impl<'a> IntoIterator for &'a Rlp<'a> { - type Item = Rlp<'a>; - type IntoIter = RlpIterator<'a>; +impl<'a> IntoIterator for &'a UntrustedRlp<'a> { + type Item = UntrustedRlp<'a>; + type IntoIter = UntrustedRlpIterator<'a>; fn into_iter(self) -> Self::IntoIter { - RlpIterator { + UntrustedRlpIterator { rlp: self, index: 0, } } } -impl<'a> Iterator for RlpIterator<'a> { - type Item = Rlp<'a>; +impl<'a> Iterator for UntrustedRlpIterator<'a> { + type Item = UntrustedRlp<'a>; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option> { let index = self.index; let result = self.rlp.at(index).ok(); self.index += 1; @@ -333,34 +335,34 @@ impl<'a> Iterator for RlpIterator<'a> { } } -/// shortcut function to decode a Rlp `&[u8]` into an object +/// shortcut function to decode a UntrustedRlp `&[u8]` into an object pub fn decode(bytes: &[u8]) -> Result where T: Decodable { - let rlp = Rlp::new(bytes); + let rlp = UntrustedRlp::new(bytes); T::decode(&rlp) } pub trait Decodable: Sized { - fn decode(rlp: &Rlp) -> Result; + fn decode(rlp: &UntrustedRlp) -> Result; } impl Decodable for T where T: FromBytes { - fn decode(rlp: &Rlp) -> Result { + fn decode(rlp: &UntrustedRlp) -> Result { match rlp.is_value() { true => BasicDecoder::read_value(rlp.bytes), - false => Err(DecoderError::RlpExpectedToBeValue), + false => Err(DecoderError::UntrustedRlpExpectedToBeValue), } } } impl Decodable for Vec where T: Decodable { - fn decode(rlp: &Rlp) -> Result { + fn decode(rlp: &UntrustedRlp) -> Result { match rlp.is_list() { true => rlp.iter().map(|rlp| T::decode(&rlp)).collect(), - false => Err(DecoderError::RlpExpectedToBeList), + false => Err(DecoderError::UntrustedRlpExpectedToBeList), } } } @@ -377,7 +379,7 @@ impl Decoder for BasicDecoder { { match bytes.first().map(|&x| x) { // rlp is too short - None => Err(DecoderError::RlpIsTooShort), + None => Err(DecoderError::UntrustedRlpIsTooShort), // single byt value Some(l @ 0...0x7f) => Ok(try!(T::from_bytes(&[l]))), // 0-55 bytes @@ -389,7 +391,7 @@ impl Decoder for BasicDecoder { let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); Ok(try!(T::from_bytes(&bytes[begin_of_value..begin_of_value + len]))) } - _ => Err(DecoderError::BadRlp), + _ => Err(DecoderError::BadUntrustedRlp), } } } @@ -497,7 +499,7 @@ impl RlpStream { } } -/// shortcut function to encode a `T: Encodable` into a Rlp `Vec` +/// shortcut function to encode a `T: Encodable` into a UntrustedRlp `Vec` pub fn encode(object: &E) -> Vec where E: Encodable { @@ -639,17 +641,16 @@ mod tests { use std::{fmt, cmp}; use std::str::FromStr; use rlp; - use rlp::{Rlp, RlpStream, Decodable}; + use rlp::{UntrustedRlp, RlpStream, Decodable}; use uint::U256; #[test] fn rlp_at() { let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; { - let rlp = Rlp::new(&data); + let rlp = UntrustedRlp::new(&data); assert!(rlp.is_list()); - let animals = - as rlp::Decodable>::decode(&rlp).unwrap(); + let animals = as rlp::Decodable>::decode(&rlp).unwrap(); assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); @@ -673,14 +674,14 @@ mod tests { fn rlp_at_err() { let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; { - let rlp = Rlp::new(&data); + let rlp = UntrustedRlp::new(&data); assert!(rlp.is_list()); let cat_err = rlp.at(0).unwrap_err(); - assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); + assert_eq!(cat_err, rlp::DecoderError::UntrustedRlpIsTooShort); let dog_err = rlp.at(1).unwrap_err(); - assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); + assert_eq!(dog_err, rlp::DecoderError::UntrustedRlpIsTooShort); } } @@ -688,7 +689,7 @@ mod tests { fn rlp_iter() { let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; { - let rlp = Rlp::new(&data); + let rlp = UntrustedRlp::new(&data); let mut iter = rlp.iter(); let cat = iter.next().unwrap(); From 6f217e730cb5ebb63ee492c2a49196e694c8819f Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 09:28:48 +0100 Subject: [PATCH 097/379] Rlp && UntrustedRlp changes --- benches/rlp.rs | 14 +++--- src/overlaydb.rs | 2 +- src/rlp.rs | 117 +++++++++++++++++++++++++---------------------- 3 files changed, 71 insertions(+), 62 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index f6a13f260..50712bc57 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -30,7 +30,7 @@ fn bench_decode_u64_value(b: &mut Bencher) { // u64 let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; let rlp = Rlp::new(&data); - let _ = u64::decode(&rlp).unwrap(); + let _ = u64::decode(&rlp); }); } @@ -54,7 +54,7 @@ fn bench_decode_u256_value(b: &mut Bencher) { 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; let rlp = Rlp::new(&data); - let _ = U256::decode(&rlp).unwrap(); + let _ = U256::decode(&rlp); }); } @@ -76,11 +76,11 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { // [ [], [[]], [ [], [[]] ] ] let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; let rlp = Rlp::new(&data); - let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); - let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); - let nested_rlp = rlp.at(2).unwrap(); - let _v2a: Vec = Decodable::decode(&nested_rlp.at(0).unwrap()).unwrap(); - let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1).unwrap()).unwrap(); + let _v0: Vec = Decodable::decode(&rlp.at(0)); + let _v1: Vec> = Decodable::decode(&rlp.at(1)); + let nested_rlp = rlp.at(2); + let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); + let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); }); } diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 99933d2cf..0f6ae3d77 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -121,7 +121,7 @@ impl OverlayDB { .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { let r = Rlp::new(d.deref()); - (Bytes::decode(&r.at(1)).unwrap(), u32::decode(&r.at(0)).unwrap()) + (Bytes::decode(&r.at(1)), u32::decode(&r.at(0))) }) } diff --git a/src/rlp.rs b/src/rlp.rs index 286eeac0d..a9532cc84 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -35,36 +35,36 @@ //! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); //! } //! -//! fn decode_value() { +//! fn decode_untrusted_value() { //! // 0x102456789abcdef //! let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; //! let rlp = UntrustedRlp::new(&data); -//! let _ = u64::decode(&rlp).unwrap(); +//! let _ = u64::decode_untrusted(&rlp).unwrap(); //! } //! -//! fn decode_string() { +//! fn decode_untrusted_string() { //! // "cat" //! let data = vec![0x83, b'c', b'a', b't']; //! let rlp = UntrustedRlp::new(&data); -//! let _ = String::decode(&rlp).unwrap(); +//! let _ = String::decode_untrusted(&rlp).unwrap(); //! } //! -//! fn decode_list() { +//! fn decode_untrusted_list() { //! // ["cat", "dog"] //! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; //! let rlp = UntrustedRlp::new(&data); -//! let _ : Vec = Decodable::decode(&rlp).unwrap(); +//! let _ : Vec = Decodable::decode_untrusted(&rlp).unwrap(); //! } //! -//! fn decode_list2() { +//! fn decode_untrusted_list2() { //! // [ [], [[]], [ [], [[]] ] ] //! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; //! let rlp = UntrustedRlp::new(&data); -//! let _v0: Vec = Decodable::decode(&rlp.at(0).unwrap()).unwrap(); -//! let _v1: Vec> = Decodable::decode(&rlp.at(1).unwrap()).unwrap(); +//! let _v0: Vec = Decodable::decode_untrusted(&rlp.at(0).unwrap()).unwrap(); +//! let _v1: Vec> = Decodable::decode_untrusted(&rlp.at(1).unwrap()).unwrap(); //! let nested_rlp = rlp.at(2).unwrap(); -//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0).unwrap()).unwrap(); -//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1).unwrap()).unwrap(); +//! let _v2a: Vec = Decodable::decode_untrusted(&nested_rlp.at(0).unwrap()).unwrap(); +//! let _v2b: Vec> = Decodable::decode_untrusted(&nested_rlp.at(1).unwrap()).unwrap(); //! } //! //! fn main() { @@ -72,10 +72,10 @@ //! encode_list(); //! encode_list2(); //! -//! decode_value(); -//! decode_string(); -//! decode_list(); -//! decode_list2(); +//! decode_untrusted_value(); +//! decode_untrusted_string(); +//! decode_untrusted_list(); +//! decode_untrusted_list2(); //! } //! ``` //! @@ -151,7 +151,7 @@ impl From for DecoderError { } } -/// Unsafe wrapper for rlp decoder. +/// Unsafe wrapper for rlp decode_untrustedr. /// /// It assumes that you know what you are doing. Doesn't bother /// you with error handling. @@ -269,7 +269,7 @@ impl<'a> UntrustedRlp<'a> { /// return first item info /// - /// TODO: move this to decoder? + /// TODO: move this to decode_untrustedr? fn item_info(bytes: &[u8]) -> Result { let item = match bytes.first().map(|&x| x) { None => return Err(DecoderError::UntrustedRlpIsTooShort), @@ -335,21 +335,30 @@ impl<'a> Iterator for UntrustedRlpIterator<'a> { } } -/// shortcut function to decode a UntrustedRlp `&[u8]` into an object -pub fn decode(bytes: &[u8]) -> Result - where T: Decodable -{ - let rlp = UntrustedRlp::new(bytes); +/// shortcut function to decoded Trusted Rlp `&[u8]` into an object +pub fn decode(bytes: &[u8]) -> T where T: Decodable { + let rlp = Rlp::new(bytes); T::decode(&rlp) } +/// shortcut function to decode UntrustedRlp `&[u8]` into an object +pub fn decode_untrusted(bytes: &[u8]) -> Result + where T: Decodable +{ + let rlp = UntrustedRlp::new(bytes); + T::decode_untrusted(&rlp) +} + pub trait Decodable: Sized { - fn decode(rlp: &UntrustedRlp) -> Result; + fn decode_untrusted(rlp: &UntrustedRlp) -> Result; + fn decode(rlp: &Rlp) -> Self { + Self::decode_untrusted(&rlp.rlp).unwrap() + } } impl Decodable for T where T: FromBytes { - fn decode(rlp: &UntrustedRlp) -> Result { + fn decode_untrusted(rlp: &UntrustedRlp) -> Result { match rlp.is_value() { true => BasicDecoder::read_value(rlp.bytes), false => Err(DecoderError::UntrustedRlpExpectedToBeValue), @@ -359,9 +368,9 @@ impl Decodable for T where T: FromBytes impl Decodable for Vec where T: Decodable { - fn decode(rlp: &UntrustedRlp) -> Result { + fn decode_untrusted(rlp: &UntrustedRlp) -> Result { match rlp.is_list() { - true => rlp.iter().map(|rlp| T::decode(&rlp)).collect(), + true => rlp.iter().map(|rlp| T::decode_untrusted(&rlp)).collect(), false => Err(DecoderError::UntrustedRlpExpectedToBeList), } } @@ -650,23 +659,23 @@ mod tests { { let rlp = UntrustedRlp::new(&data); assert!(rlp.is_list()); - let animals = as rlp::Decodable>::decode(&rlp).unwrap(); + let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); assert!(cat.is_value()); assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode(&cat).unwrap(), "cat".to_string()); + assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); assert!(dog.is_value()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); - assert_eq!(String::decode(&dog).unwrap(), "dog".to_string()); + assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_value()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); } } @@ -865,17 +874,17 @@ mod tests { struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; - fn run_decode_tests(tests: Vec>) + fn run_decode_untrusted_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { for t in &tests { - let res: T = rlp::decode(&t.1).unwrap(); + let res: T = rlp::decode_untrusted(&t.1).unwrap(); assert_eq!(res, t.0); } } #[test] - fn decode_u8() { + fn decode_untrusted_u8() { let tests = vec![ DTestPair(0u8, vec![0u8]), DTestPair(15, vec![15]), @@ -885,41 +894,41 @@ mod tests { DTestPair(0x80, vec![0x81, 0x80]), DTestPair(0xff, vec![0x81, 0xff]), ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_u16() { + fn decode_untrusted_u16() { let tests = vec![ DTestPair(0u16, vec![0u8]), DTestPair(0x100, vec![0x82, 0x01, 0x00]), DTestPair(0xffff, vec![0x82, 0xff, 0xff]), ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_u32() { + fn decode_untrusted_u32() { let tests = vec![ DTestPair(0u32, vec![0u8]), DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_u64() { + fn decode_untrusted_u64() { let tests = vec![ DTestPair(0u64, vec![0u8]), DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_u256() { + fn decode_untrusted_u256() { let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), DTestPair(U256::from(0xffffffffu64), @@ -931,11 +940,11 @@ mod tests { 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_str() { + fn decode_untrusted_str() { let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), DTestPair("Marek".to_string(), @@ -950,11 +959,11 @@ mod tests { b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', b'e', b'l', b'i', b't'])]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_address() { + fn decode_untrusted_address() { use hash::*; let tests = vec![ @@ -963,41 +972,41 @@ mod tests { 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, 0xb3, 0x7d, 0x11, 0x06]) ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_vector_u8() { + fn decode_untrusted_vector_u8() { let tests = vec![ DTestPair(vec![] as Vec, vec![0xc0]), DTestPair(vec![15u8], vec![0xc1, 0x0f]), DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_vector_u64() { + fn decode_untrusted_vector_u64() { let tests = vec![ DTestPair(vec![], vec![0xc0]), DTestPair(vec![15u64], vec![0xc1, 0x0f]), DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_vector_str() { + fn decode_untrusted_vector_str() { let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } #[test] - fn decode_vector_of_vectors_str() { + fn decode_untrusted_vector_of_vectors_str() { let tests = vec![DTestPair(vec![vec!["cat".to_string()]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_decode_tests(tests); + run_decode_untrusted_tests(tests); } } From 73566f071da019d29389262863cf0f83ff31ecf3 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 09:37:07 +0100 Subject: [PATCH 098/379] updated rlp docs --- src/rlp.rs | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index a9532cc84..ef1232c55 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1,13 +1,14 @@ -//! UntrustedRlp serialization module +//! Rlp serialization module //! //! Types implementing `Endocable` and `Decodable` traits -//! can be easily coverted to and from rlp +//! can be easily coverted to and from rlp. +//! Trusted rlp should be decoded with `Rlp`, untrusted with `UntrustedRlp`. //! //! # Examples: //! //! ```rust //! extern crate ethcore_util; -//! use ethcore_util::rlp::{UntrustedRlp, RlpStream, Decodable}; +//! use ethcore_util::rlp::{Rlp, UntrustedRlp, RlpStream, Decodable}; //! //! fn encode_value() { //! // 1029 @@ -35,11 +36,11 @@ //! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); //! } //! -//! fn decode_untrusted_value() { +//! fn decode_value() { //! // 0x102456789abcdef //! let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; -//! let rlp = UntrustedRlp::new(&data); -//! let _ = u64::decode_untrusted(&rlp).unwrap(); +//! let rlp = Rlp::new(&data); +//! let _ = u64::decode(&rlp); //! } //! //! fn decode_untrusted_string() { @@ -49,22 +50,22 @@ //! let _ = String::decode_untrusted(&rlp).unwrap(); //! } //! -//! fn decode_untrusted_list() { +//! fn decode_list() { //! // ["cat", "dog"] //! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -//! let rlp = UntrustedRlp::new(&data); -//! let _ : Vec = Decodable::decode_untrusted(&rlp).unwrap(); +//! let rlp = Rlp::new(&data); +//! let _ : Vec = Decodable::decode(&rlp); //! } //! -//! fn decode_untrusted_list2() { +//! fn decode_list2() { //! // [ [], [[]], [ [], [[]] ] ] //! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; -//! let rlp = UntrustedRlp::new(&data); -//! let _v0: Vec = Decodable::decode_untrusted(&rlp.at(0).unwrap()).unwrap(); -//! let _v1: Vec> = Decodable::decode_untrusted(&rlp.at(1).unwrap()).unwrap(); -//! let nested_rlp = rlp.at(2).unwrap(); -//! let _v2a: Vec = Decodable::decode_untrusted(&nested_rlp.at(0).unwrap()).unwrap(); -//! let _v2b: Vec> = Decodable::decode_untrusted(&nested_rlp.at(1).unwrap()).unwrap(); +//! let rlp = Rlp::new(&data); +//! let _v0: Vec = Decodable::decode(&rlp.at(0)); +//! let _v1: Vec> = Decodable::decode(&rlp.at(1)); +//! let nested_rlp = rlp.at(2); +//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); +//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); //! } //! //! fn main() { @@ -72,10 +73,10 @@ //! encode_list(); //! encode_list2(); //! -//! decode_untrusted_value(); +//! decode_value(); //! decode_untrusted_string(); -//! decode_untrusted_list(); -//! decode_untrusted_list2(); +//! decode_list(); +//! decode_list2(); //! } //! ``` //! From 5c975d06cc428eb843cf35fc336611a7167ad38f Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 29 Nov 2015 11:50:28 +0100 Subject: [PATCH 099/379] Started networking code --- src/network/host.rs | 634 ++++++++++++++++++++++++++++++++++++++++++++ src/network/mod.rs | 2 + 2 files changed, 636 insertions(+) create mode 100644 src/network/host.rs create mode 100644 src/network/mod.rs diff --git a/src/network/host.rs b/src/network/host.rs new file mode 100644 index 000000000..c945b0e57 --- /dev/null +++ b/src/network/host.rs @@ -0,0 +1,634 @@ +#![allow(dead_code)] //TODO: remove this after everything is done +//TODO: remove all unwraps +use std::net::{SocketAddr, AddrParseError}; +use std::collections::{HashSet, HashMap, BTreeMap}; +use std::hash::{Hash, Hasher}; +use std::cell::{RefCell}; +use std::ops::{DerefMut}; +use std::str::{FromStr}; +use mio::*; +use mio::util::{Slab}; +use mio::tcp::*; +use mio::udp::*; +use hash::*; +use bytes::*; +use time::Tm; +use error::EthcoreError; + +const DEFAULT_PORT: u16 = 30303; + +const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. +const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia]. +const NODE_BINS: u32 = ADDRESS_BITS - 1; ///< Size of m_state (excludes root, which is us). +const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover) +const MAX_CONNECTIONS: usize = 1024; +const IDEAL_PEERS:u32 = 10; + +const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. +const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. + +type NodeId = H512; +type PublicKey = H512; +type SecretKey = H256; + +#[derive(Debug)] +struct NetworkConfiguration { + listen_address: SocketAddr, + public_address: SocketAddr, + no_nat: bool, + no_discovery: bool, + pin: bool, +} + +impl NetworkConfiguration { + fn new() -> NetworkConfiguration { + NetworkConfiguration { + listen_address: SocketAddr::from_str("0.0.0.0:30303").unwrap(), + public_address: SocketAddr::from_str("0.0.0.0:30303").unwrap(), + no_nat: false, + no_discovery: false, + pin: false + } + } +} + +#[derive(Debug)] +struct NodeEndpoint { + address: SocketAddr, + udp_port: u16 +} + +impl NodeEndpoint { + fn new(address: SocketAddr) -> NodeEndpoint { + NodeEndpoint { + address: address, + udp_port: address.port() + } + } + fn from_str(address: &str) -> Result { + let address = try!(SocketAddr::from_str(address)); + Ok(NodeEndpoint { + address: address, + udp_port: address.port() + }) + } +} + +#[derive(Debug)] +pub enum AddressError { + AddrParseError(AddrParseError), + NodeIdParseError(EthcoreError) +} + +impl From for AddressError { + fn from(err: AddrParseError) -> AddressError { + AddressError::AddrParseError(err) + } +} +impl From for AddressError { + fn from(err: EthcoreError) -> AddressError { + AddressError::NodeIdParseError(err) + } +} + +#[derive(PartialEq, Eq, Copy, Clone)] +enum PeerType { + Required, + Optional +} + +struct Node { + id: NodeId, + endpoint: NodeEndpoint, + peer_type: PeerType, + last_attempted: Option, + confirmed: bool, +} + +impl FromStr for Node { + type Err = AddressError; + fn from_str(s: &str) -> Result { + let (id, endpoint) = if &s[..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { + (try!(NodeId::from_str(&s[8..128])), try!(NodeEndpoint::from_str(&s[137..]))) + } + else { + (NodeId::new(), try!(NodeEndpoint::from_str(s))) + }; + + Ok(Node { + id: id, + endpoint: endpoint, + peer_type: PeerType::Optional, + last_attempted: None, + confirmed: false + }) + } +} + +impl Node { + fn new(id: NodeId, address: SocketAddr, t:PeerType) -> Node { + Node { + id: id, + endpoint: NodeEndpoint::new(address), + peer_type: t, + last_attempted: None, + confirmed: false + } + } +} + +impl PartialEq for Node { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} +impl Eq for Node { } + +impl Hash for Node { + fn hash(&self, state: &mut H) where H: Hasher { + self.id.hash(state) + } +} + +struct NodeBucket { + distance: u32, + nodes: Vec +} + +impl NodeBucket { + fn new(distance: u32) -> NodeBucket { + NodeBucket { + distance: distance, + nodes: Vec::new() + } + } +} + +struct Connection { + socket: TcpStream, + send_queue: Vec, +} + +impl Connection { + fn new(socket: TcpStream) -> Connection { + Connection { + socket: socket, + send_queue: Vec::new(), + } + } +} + +#[derive(PartialEq, Eq)] +enum HandshakeState { + New, + AckAuth, + WriteHello, + ReadHello, + StartSession, +} + +struct Handshake { + id: NodeId, + connection: Connection, + state: HandshakeState, +} + +impl Handshake { + fn new(id: NodeId, socket: TcpStream) -> Handshake { + Handshake { + id: id, + connection: Connection::new(socket), + state: HandshakeState::New + } + } +} + +struct Peer { + id: NodeId, + connection: Connection, +} + +struct FindNodePacket; + +impl FindNodePacket { + fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { + FindNodePacket + } + fn sign(&mut self, _secret: &SecretKey) { + } + + fn send(& self, _socket: &mut UdpSocket) { + } +} + +// Tokens +const TCP_ACCEPT: usize = 1; +const IDLE: usize = 3; +const NODETABLE_RECEIVE: usize = 4; +const NODETABLE_MAINTAIN: usize = 5; +const NODETABLE_DISCOVERY: usize = 6; +const FIRST_CONNECTION: usize = 7; +const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; +const FIRST_HANDSHAKE: usize = FIRST_CONNECTION + MAX_CONNECTIONS; +const LAST_HANDSHAKE: usize = FIRST_HANDSHAKE + MAX_CONNECTIONS - 1; + +pub enum HostMessage { + Shutdown +} + +pub struct Host { + secret: SecretKey, + node: Node, + sender: Sender, + config: NetworkConfiguration, + udp_socket: UdpSocket, + listener: TcpListener, + peers: Slab, + connecting: Slab, + discovery_round: u16, + discovery_id: NodeId, + discovery_nodes: HashSet, + node_buckets: Vec, + nodes: HashMap, + idle_timeout: Timeout, +} + +impl Host { + pub fn start() { + let config = NetworkConfiguration::new(); + /* + match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { + Some(iface) => config.public_address = iface.addr.unwrap(), + None => warn!("No public network interface"), + } + */ + + let addr = config.listen_address; + // Setup the server socket + let listener = TcpListener::bind(&addr).unwrap(); + // Create an event loop + let mut event_loop = EventLoop::new().unwrap(); + let sender = event_loop.channel(); + // Start listening for incoming connections + event_loop.register_opt(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); + // Setup the client socket + //let sock = TcpStream::connect(&addr).unwrap(); + // Register the socket + //self.event_loop.register_opt(&sock, CLIENT, EventSet::readable(), PollOpt::edge()).unwrap(); + let idle_timeout = event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay + // open the udp socket + let udp_socket = UdpSocket::bound(&addr).unwrap(); + event_loop.register_opt(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); + event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); + + let mut host = Host { + secret: SecretKey::new(), + node: Node::new(NodeId::new(), config.public_address.clone(), PeerType::Required), + config: config, + sender: sender, + udp_socket: udp_socket, + listener: listener, + peers: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), + connecting: Slab::new_starting_at(Token(FIRST_HANDSHAKE), MAX_CONNECTIONS), + discovery_round: 0, + discovery_id: NodeId::new(), + discovery_nodes: HashSet::new(), + node_buckets: (0..NODE_BINS).map(|x| NodeBucket::new(x)).collect(), + nodes: HashMap::new(), + idle_timeout: idle_timeout + }; + + + host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); + host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02dd@gav.ethdev.com:30303"); + host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163@52.16.188.185:30303"); + host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e03@54.207.93.166:30303"); + host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254@92.51.165.126:30303"); + + event_loop.run(&mut host).unwrap(); + } + + fn stop(&mut self) { + } + + fn have_network(&mut self) -> bool { + true + } + + fn add_node(&mut self, id: &str) { + match Node::from_str(id) { + Err(e) => { warn!("Could not add node: {:?}", e); }, + Ok(n) => { self.nodes.insert(n.id.clone(), n); } + } + } + + fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { + self.discovery_round = 0; + self.discovery_id.randomize(); + self.discovery_nodes.clear(); + self.discover(event_loop); + } + + fn discover(&mut self, event_loop: &mut EventLoop) { + if self.discovery_round == DISCOVERY_MAX_STEPS + { + debug!("Restarting discovery"); + self.start_node_discovery(event_loop); + return; + } + let mut tried_count = 0; + { + let nearest = Host::nearest_node_entries(&self.node.id, &self.discovery_id, &self.node_buckets).into_iter(); + let nodes = RefCell::new(&mut self.discovery_nodes); + let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); + for r in nearest { + //let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id); + //p.sign(&self.secret); + //p.send(&mut self.udp_socket); + let mut borrowed = nodes.borrow_mut(); + borrowed.deref_mut().insert(r.clone()); + tried_count += 1; + } + } + + if tried_count == 0 + { + debug!("Restarting discovery"); + self.start_node_discovery(event_loop); + return; + } + self.discovery_round += 1; + event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap(); + } + + fn distance(a: &NodeId, b: &NodeId) -> u32 { + //TODO: + //u256 d = sha3(_a) ^ sha3(_b); + let mut d: NodeId = NodeId::new(); + for i in 0..32 { + d[i] = a[i] ^ b[i]; + } + + let mut ret:u32 = 0; + for i in 0..32 { + let mut v: u8 = d[i]; + while v != 0 { + v >>= 1; + ret += 1; + } + } + ret + } + + fn nearest_node_entries<'a>(source: &NodeId, target: &NodeId, buckets: &'a Vec) -> Vec<&'a NodeId> + { + // send ALPHA FindNode packets to nodes we know, closest to target + const LAST_BIN: u32 = NODE_BINS - 1; + let mut head = Host::distance(source, target); + let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; + + let mut found: BTreeMap> = BTreeMap::new(); + let mut count = 0; + + // if d is 0, then we roll look forward, if last, we reverse, else, spread from d + if head > 1 && tail != LAST_BIN { + while head != tail && head < NODE_BINS && count < BUCKET_SIZE + { + for n in buckets[head as usize].nodes.iter() + { + if count < BUCKET_SIZE { + count += 1; + found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + if count < BUCKET_SIZE && tail != 0 { + for n in buckets[tail as usize].nodes.iter() { + if count < BUCKET_SIZE { + count += 1; + found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + } + + head += 1; + if tail > 0 { + tail -= 1; + } + } + } + else if head < 2 { + while head < NODE_BINS && count < BUCKET_SIZE { + for n in buckets[head as usize].nodes.iter() { + if count < BUCKET_SIZE { + count += 1; + found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + head += 1; + } + } + else { + while tail > 0 && count < BUCKET_SIZE { + for n in buckets[tail as usize].nodes.iter() { + if count < BUCKET_SIZE { + count += 1; + found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + tail -= 1; + } + } + + let mut ret:Vec<&NodeId> = Vec::new(); + for (_, nodes) in found { + for n in nodes { + if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ { + ret.push(n); + } + } + } + ret + } + + fn maintain_network(&mut self, event_loop: &mut EventLoop) { + self.keep_alive(); + self.connect_peers(event_loop); + } + + fn have_session(&self, id: &NodeId) -> bool { + self.peers.iter().any(|h| h.id.eq(&id)) + } + + fn connecting_to(&self, id: &NodeId) -> bool { + self.connecting.iter().any(|h| h.id.eq(&id)) + } + + fn connect_peers(&mut self, event_loop: &mut EventLoop) { + + struct NodeInfo { + id: NodeId, + peer_type: PeerType + } + + let mut to_connect: Vec = Vec::new(); + + let mut req_conn = 0; + for n in self.node_buckets.iter().flat_map(|n| &n.nodes).map(|id| NodeInfo { id: id.clone(), peer_type: self.nodes.get(id).unwrap().peer_type}) { + let connected = self.have_session(&n.id) || self.connecting_to(&n.id); + let required = n.peer_type == PeerType::Required; + if connected && required { + req_conn += 1; + } + else if !connected && (!self.config.pin || required) { + to_connect.push(n); + } + } + + for n in to_connect.iter() { + if n.peer_type == PeerType::Required { + if req_conn < IDEAL_PEERS { + self.connect_peer(&n.id, event_loop); + } + req_conn += 1; + } + } + + if !self.config.pin + { + let pending_count = 0; //TODO: + let peer_count = 0; + let mut open_slots = IDEAL_PEERS - peer_count - pending_count + req_conn; + if open_slots > 0 { + for n in to_connect.iter() { + if n.peer_type == PeerType::Optional && open_slots > 0 { + open_slots -= 1; + self.connect_peer(&n.id, event_loop); + } + } + } + } + } + + fn connect_peer(&mut self, id: &NodeId, event_loop: &mut EventLoop) { + if self.have_session(id) + { + warn!("Aborted connect. Node already connected."); + return; + } + if self.connecting_to(id) + { + warn!("Aborted connect. Node already connecting."); + return; + } + let node = self.nodes.get_mut(id).unwrap(); + node.last_attempted = Some(::time::now()); + + + //blog(NetConnect) << "Attempting connection to node" << _p->id << "@" << ep << "from" << id(); + let socket = match TcpStream::connect(&node.endpoint.address) { + Ok(socket) => socket, + Err(_) => { + warn!("Cannot connect to node"); + return; + } + }; + let handshake = Handshake::new(id.clone(), socket); + match self.connecting.insert(handshake) { + Ok(token) => event_loop.register_opt(&self.connecting[token].connection.socket, token, EventSet::all(), PollOpt::edge()).unwrap(), + Err(_) => warn!("Max connections reached") + }; + } + + fn keep_alive(&mut self) { + } + + + + fn accept(&mut self, _event_loop: &mut EventLoop) { + warn!(target "net", "accept"); + } + + fn start_handshake(&mut self, token: Token, _event_loop: &mut EventLoop) { + let handshake = match self.handshakes.get(&token) { + Some(h) => h, + None => { + warn!(target "net", "Received event for unknown handshake"); + return; + } + }; + + + + + } + + fn read_handshake(&mut self, _event_loop: &mut EventLoop) { + warn!(target "net", "accept"); + } + + fn read_connection(&mut self, _event_loop: &mut EventLoop) { + } + + fn write_connection(&mut self, _event_loop: &mut EventLoop) { + } +} + +impl Handler for Host { + type Timeout = Token; + type Message = HostMessage; + + fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { + if events.is_readable() { + match token.as_usize() { + TCP_ACCEPT => self.accept(event_loop), + IDLE => self.maintain_network(event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.read_connection(event_loop), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.read_handshake(event_loop), + NODETABLE_RECEIVE => {}, + _ => panic!("Received unknown readable token"), + } + } + else if events.is_writable() { + match token.as_usize() { + FIRST_CONNECTION ... LAST_CONNECTION => self.write_connection(event_loop), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.start_handshake(event_loop), + _ => panic!("Received unknown writable token"), + } + } + } + + fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { + match token.as_usize() { + IDLE => self.maintain_network(event_loop), + NODETABLE_DISCOVERY => {}, + NODETABLE_MAINTAIN => {}, + _ => panic!("Received unknown timer token"), + } + } +} + + +#[cfg(test)] +mod tests { + use network::host::Host; + #[test] + #[ignore] + fn net_connect() { + let _ = Host::start(); + } +} + + + diff --git a/src/network/mod.rs b/src/network/mod.rs new file mode 100644 index 000000000..917d79464 --- /dev/null +++ b/src/network/mod.rs @@ -0,0 +1,2 @@ +extern crate mio; +pub mod host; From 96e1580ddfad675ca3a4e908888f1bc6c39275b1 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 12:22:35 +0100 Subject: [PATCH 100/379] triehash, hex_prefix_encode --- src/lib.rs | 1 + src/triehash.rs | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/triehash.rs diff --git a/src/lib.rs b/src/lib.rs index 27c927b2c..098e42928 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ pub mod overlaydb; pub mod math; pub mod chainfilter; pub mod crypto; +pub mod triehash; //pub mod network; diff --git a/src/triehash.rs b/src/triehash.rs new file mode 100644 index 000000000..47665933b --- /dev/null +++ b/src/triehash.rs @@ -0,0 +1,90 @@ +//! Generete trie root + +//use std::collections::HashMap; +//use hash::*; +//use rlp; + +/// +/// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. +/// +/// The "termination marker" and "leaf-node" specifier are completely equivalent. +/// +/// Input values are in range `[0, 0xf]`. +/// +/// ```markdown +/// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 +/// [0,1,2,3,4,5] 0x00012345 // 6 > 4 +/// [1,2,3,4,5] 0x112345 // 5 > 3 +/// [0,0,1,2,3,4] 0x00001234 // 6 > 3 +/// [0,1,2,3,4] 0x101234 // 5 > 3 +/// [1,2,3,4] 0x001234 // 4 > 3 +/// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 +/// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 +/// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 +/// [1,2,3,4,5,T] 0x312345 // 5 > 3 +/// [1,2,3,4,T] 0x201234 // 4 > 3 +/// ``` +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::triehash::*; +/// +/// fn main() { +/// let v = vec![0, 0, 1, 2, 3, 4, 5]; +/// let e = vec![0x10, 0x01, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, false); +/// assert_eq!(h, e); +/// +/// let v = vec![0, 1, 2, 3, 4, 5]; +/// let e = vec![0x00, 0x01, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, false); +/// assert_eq!(h, e); +/// +/// let v = vec![0, 1, 2, 3, 4, 5]; +/// let e = vec![0x20, 0x01, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, true); +/// assert_eq!(h, e); +/// +/// let v = vec![1, 2, 3, 4, 5]; +/// let e = vec![0x31, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, true); +/// assert_eq!(h, e); +/// +/// let v = vec![1, 2, 3, 4]; +/// let e = vec![0x00, 0x12, 0x34]; +/// let h = hex_prefix_encode(&v, false); +/// assert_eq!(h, e); +/// } +/// ``` +/// +pub fn hex_prefix_encode(hex: &[u8], leaf: bool) -> Vec { + let inlen = hex.len(); + let oddness_factor = inlen % 2; + // next even number divided by two + let reslen = (inlen + 2) >> 1; + let mut res = vec![]; + res.reserve(reslen); + + let first_byte = { + let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; + if oddness_factor == 1 { + bits += hex[0]; + } + bits + }; + + res.push(first_byte); + + let mut offset = oddness_factor; + while offset < inlen { + let byte = (hex[offset] << 4) + hex[offset + 1]; + res.push(byte); + offset += 2; + } + + res +} + +#[cfg(test)] +mod tests { +} From 1f6ae5646863618f190a16a37ffdc68f2e7502ae Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 12:39:37 +0100 Subject: [PATCH 101/379] as_nibbles --- src/triehash.rs | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 47665933b..5da09e747 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -4,7 +4,6 @@ //use hash::*; //use rlp; -/// /// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. /// /// The "termination marker" and "leaf-node" specifier are completely equivalent. @@ -57,8 +56,8 @@ /// } /// ``` /// -pub fn hex_prefix_encode(hex: &[u8], leaf: bool) -> Vec { - let inlen = hex.len(); +pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { + let inlen = nibbles.len(); let oddness_factor = inlen % 2; // next even number divided by two let reslen = (inlen + 2) >> 1; @@ -68,7 +67,7 @@ pub fn hex_prefix_encode(hex: &[u8], leaf: bool) -> Vec { let first_byte = { let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; if oddness_factor == 1 { - bits += hex[0]; + bits += nibbles[0]; } bits }; @@ -77,7 +76,7 @@ pub fn hex_prefix_encode(hex: &[u8], leaf: bool) -> Vec { let mut offset = oddness_factor; while offset < inlen { - let byte = (hex[offset] << 4) + hex[offset + 1]; + let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; res.push(byte); offset += 2; } @@ -85,6 +84,28 @@ pub fn hex_prefix_encode(hex: &[u8], leaf: bool) -> Vec { res } +/// Converts slice of bytes to nibbles. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::triehash::*; +/// +/// fn main () { +/// let v = vec![0x31, 0x23, 0x45]; +/// let e = vec![3, 1, 2, 3, 4, 5]; +/// assert_eq!(as_nibbles(&v), e); +/// } +/// ``` +pub fn as_nibbles(bytes: &[u8]) -> Vec { + let mut res = vec![]; + res.reserve(bytes.len() * 2); + for i in 0..bytes.len() { + res.push(bytes[i] >> 4); + res.push((bytes[i] << 4) >> 4); + } + res +} + #[cfg(test)] mod tests { } From 7e58f12712e4c7ce953fdeb12210a1808645dcf3 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 15:41:29 +0100 Subject: [PATCH 102/379] triehash in progress --- src/rlp.rs | 29 ++++++++++++++++------ src/triehash.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++++--- src/vector.rs | 4 ++- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index ef1232c55..5b08aca2b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -446,7 +446,7 @@ impl RlpStream { stream } - /// apends value to the end of stream, chainable + /// Apends value to the end of stream, chainable. pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { @@ -454,13 +454,13 @@ impl RlpStream { object.encode(&mut self.encoder); // if list is finished, prepend the length - self.try_to_finish(); + self.try_to_finish(1); // return chainable self self } - /// declare appending the list of given size + /// Declare appending the list of given size, chainable. pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { // push new list let position = self.encoder.bytes.len(); @@ -468,7 +468,7 @@ impl RlpStream { 0 => { // we may finish, if the appended list len is equal 0 self.encoder.bytes.push(0xc0u8); - self.try_to_finish(); + self.try_to_finish(1); } _ => self.unfinished_lists.push_back(ListInfo::new(position, len)), } @@ -477,6 +477,18 @@ impl RlpStream { self } + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. + pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a RlpStream { + // push raw items + self.encoder.bytes.extend(bytes); + + // try to finish and prepend the length + self.try_to_finish(item_count); + + // return chainable self + self + } + /// return true if stream is ready pub fn is_finished(&self) -> bool { self.unfinished_lists.back().is_none() @@ -491,11 +503,14 @@ impl RlpStream { } /// try to finish lists - fn try_to_finish(&mut self) -> () { + fn try_to_finish(&mut self, inserted_items: usize) -> () { let should_finish = match self.unfinished_lists.back_mut() { None => false, Some(ref mut x) => { - x.current += 1; + x.current += inserted_items; + if x.current > x.max { + panic!("You cannot append more items then you expect!"); + } x.current == x.max } }; @@ -504,7 +519,7 @@ impl RlpStream { let x = self.unfinished_lists.pop_back().unwrap(); let len = self.encoder.bytes.len() - x.position; self.encoder.insert_list_len_at_pos(len, x.position); - self.try_to_finish(); + self.try_to_finish(1); } } } diff --git a/src/triehash.rs b/src/triehash.rs index 5da09e747..7059cb5f9 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,8 +1,11 @@ //! Generete trie root -//use std::collections::HashMap; -//use hash::*; -//use rlp; +use std::collections::BTreeMap; +use std::cmp; +use hash::*; +use sha3::*; +use rlp; +use rlp::RlpStream; /// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. /// @@ -106,6 +109,63 @@ pub fn as_nibbles(bytes: &[u8]) -> Vec { res } +struct NibblePair { + nibble: Vec, + data: Vec +} + +pub fn ordered_trie_root(data: Vec>) -> H256 { + let vec: Vec = data + // first put elements into btree to sort them by nibbles + // optimize it later + .into_iter() + .fold(BTreeMap::new(), | mut acc, vec | { + let len = acc.len(); + acc.insert(as_nibbles(&rlp::encode(&len)), vec); + acc + }) + // then move them to a vector + .into_iter() + .map(|(k, v)| NibblePair { nibble: k, data: v } ) + .collect(); + + let out = match vec.len() { + 0 => rlp::encode(&""), + _ => { + let mut stream = RlpStream::new(); + hash256rlp(&vec, 0, &mut stream); + stream.out().unwrap() + } + }; + + out.sha3() +} + +fn shared_prefix_length(v1: &[T], v2: &[T]) -> usize where T: Eq { + let len = cmp::min(v1.len(), v2.len()); + (0..len).take_while(|&i| v1[i] == v2[i]).count() +} + +fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { + match vec.len() { + 0 => stream.append(&""), + 1 => stream.append_list(2).append(&hex_prefix_encode(&vec[0].nibble, true)).append(&vec[0].data), + _ => { + let shared_prefix = vec.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first string and each successive + .fold(usize::max_value(), | acc, pair | cmp::min(shared_prefix_length(&vec[0].nibble, &pair.nibble), acc) ); + //match shared_prefix > pre_len { + + //true => hex_prefix_encode(&vec[0].nibble + //} + panic!(); + + } + }; +} + #[cfg(test)] mod tests { } diff --git a/src/vector.rs b/src/vector.rs index ce891beba..7032335cc 100644 --- a/src/vector.rs +++ b/src/vector.rs @@ -1,4 +1,3 @@ - use std::ptr; pub trait InsertSlice { @@ -32,3 +31,6 @@ impl InsertSlice for Vec { } } } + +pub trait SharedPreifx { +} From b334a8f824708543364b817b4131dbeef0c69742 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 15:50:33 +0100 Subject: [PATCH 103/379] Return lookup by reference. --- src/hashdb.rs | 2 +- src/lib.rs | 1 + src/memorydb.rs | 32 ++++++++++++++++++++++-- src/overlaydb.rs | 39 +++++++++++++++-------------- src/trie.rs | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 22 deletions(-) create mode 100644 src/trie.rs diff --git a/src/hashdb.rs b/src/hashdb.rs index 48f5182bc..eb41d23df 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -17,7 +17,7 @@ pub trait HashDB { /// assert_eq!(m.lookup(&hash).unwrap(), hello_bytes); /// } /// ``` - fn lookup(&self, key: &H256) -> Option; + fn lookup(&self, key: &H256) -> Option<&[u8]>; /// Check for the existance of a hash-key. /// diff --git a/src/lib.rs b/src/lib.rs index 098e42928..983dc322c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,7 @@ pub mod math; pub mod chainfilter; pub mod crypto; pub mod triehash; +pub mod trie; //pub mod network; diff --git a/src/memorydb.rs b/src/memorydb.rs index cbb9e426b..58dc980ca 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -100,12 +100,23 @@ impl MemoryDB { mem::swap(&mut self.data, &mut data); data } + + pub fn denote(&self, key: &H256, value: Bytes) -> &(Bytes, i32) { + if self.data.get(&key) == None { + unsafe { + let p = &self.data as *const HashMap; + let mp = p as *mut HashMap; + (*mp).insert(key.clone(), (value, 0)); + } + } + self.data.get(key).unwrap() + } } impl HashDB for MemoryDB { - fn lookup(&self, key: &H256) -> Option { + fn lookup(&self, key: &H256) -> Option<&[u8]> { match self.data.get(key) { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + Some(&(ref d, rc)) if rc > 0 => Some(d), _ => None } } @@ -143,3 +154,20 @@ impl HashDB for MemoryDB { } } +#[test] +fn memorydb_denote() { + let mut m = MemoryDB::new(); + let hello_bytes = b"Hello world!"; + let hash = m.insert(hello_bytes); + assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); + + for i in 0..1000 { + let r = H256::random(); + let k = r.sha3(); + let &(ref v, ref rc) = m.denote(&k, r.bytes().to_vec()); + assert_eq!(v, &r.bytes()); + assert_eq!(*rc, 0); + } + + assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); +} \ No newline at end of file diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 0f6ae3d77..86ad266c9 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -8,6 +8,7 @@ use hashdb::*; use memorydb::*; use std::ops::*; use std::sync::*; +use std::cell::*; use std::env; use rocksdb::{DB, Writable}; @@ -135,19 +136,19 @@ impl OverlayDB { } impl HashDB for OverlayDB { - fn lookup(&self, key: &H256) -> Option { + fn lookup(&self, key: &H256) -> Option<&[u8]> { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. let k = self.overlay.raw(key); match k { - Some(&(ref d, rc)) if rc > 0 => Some(d.clone()), + Some(&(ref d, rc)) if rc > 0 => Some(d), _ => { let memrc = k.map(|&(_, rc)| rc).unwrap_or(0); match self.payload(key) { Some(x) => { let (d, rc) = x; if rc as i32 + memrc > 0 { - Some(d) + Some(&self.overlay.denote(key, d).0) } else { None @@ -193,7 +194,7 @@ impl HashDB for OverlayDB { fn overlaydb_overlay_insert_and_kill() { let mut trie = OverlayDB::new_temp(); let h = trie.insert(b"hello world"); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); trie.kill(&h); assert_eq!(trie.lookup(&h), None); } @@ -202,11 +203,11 @@ fn overlaydb_overlay_insert_and_kill() { fn overlaydb_backing_insert_revert() { let mut trie = OverlayDB::new_temp(); let h = trie.insert(b"hello world"); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); trie.revert(); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); } #[test] @@ -230,7 +231,7 @@ fn overlaydb_backing_kill_revert() { trie.kill(&h); assert_eq!(trie.lookup(&h), None); trie.revert(); - assert_eq!(trie.lookup(&h), Some(b"hello world".to_vec())); + assert_eq!(trie.lookup(&h).unwrap(), b"hello world"); } #[test] @@ -248,29 +249,29 @@ fn overlaydb_negative() { fn overlaydb_complex() { let mut trie = OverlayDB::new_temp(); let hfoo = trie.insert(b"foo"); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); let hbar = trie.insert(b"bar"); - assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + assert_eq!(trie.lookup(&hbar).unwrap(), b"bar"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); - assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); + assert_eq!(trie.lookup(&hbar).unwrap(), b"bar"); trie.insert(b"foo"); // two refs - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); - assert_eq!(trie.lookup(&hbar), Some(b"bar".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); + assert_eq!(trie.lookup(&hbar).unwrap(), b"bar"); trie.kill(&hbar); // zero refs - delete assert_eq!(trie.lookup(&hbar), None); trie.kill(&hfoo); // one ref - keep - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.kill(&hfoo); // zero ref - would delete, but... assert_eq!(trie.lookup(&hfoo), None); trie.insert(b"foo"); // one ref - keep after all. - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.commit().unwrap(); - assert_eq!(trie.lookup(&hfoo), Some(b"foo".to_vec())); + assert_eq!(trie.lookup(&hfoo).unwrap(), b"foo"); trie.kill(&hfoo); // zero ref - delete assert_eq!(trie.lookup(&hfoo), None); trie.commit().unwrap(); // diff --git a/src/trie.rs b/src/trie.rs new file mode 100644 index 000000000..41c8f58ea --- /dev/null +++ b/src/trie.rs @@ -0,0 +1,64 @@ +use memorydb::*; +use hashdb::*; +use hash::*; +use rlp::*; +use bytes::*; + +pub const NULL_RLP: [u8; 1] = [0x80; 1]; +pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); + +/*lazy_static! { + pub static ref NULL_RLP: Bytes = { let mut r = RlpStream::new(); r.append(&""); r.out().unwrap() }; + pub static ref SHA3_NULL_RLP: H256 = { use sha3::Hashable; NULL_RLP.sha3() }; +}*/ + +pub trait Trie { + fn root(&self) -> &H256; + fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } + + // TODO: consider returning &[u8]... +/* fn at(key: &[u8]) -> Option; + fn insert(key: &[u8], value: &[u8]); + fn remove(key: &[u8]); + fn contains(key: &[u8]) -> bool;*/ +} + +pub struct TrieDB { + db: Box, + root: H256, +} + +impl TrieDB { + pub fn new(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } } + + pub fn new_boxed(db_box: Box) -> Self { TrieDB{ db: db_box, root: H256::new() } } + + pub fn new_memory() -> Self { TrieDB{ db: Box::new(MemoryDB::new()), root: H256::new() } } + + pub fn init(&mut self) { self.insert_root(&NULL_RLP); } + + pub fn db(&self) -> &HashDB { self.db.as_ref() } + +// pub fn db_mut(&mut self) -> &mut HashDB { self.db.as } + + fn insert_root(&mut self, root_data: &[u8]) { self.root = self.db.insert(root_data); } + +} + +impl Trie for TrieDB { + fn root(&self) -> &H256 { &self.root } +} + +#[test] +fn it_works() { + use overlaydb::*; + + (&[1, 2, 3]).starts_with(&[1, 2]); + + let mut t = TrieDB::new(OverlayDB::new_temp()); + t.init(); + assert_eq!(*t.root(), SHA3_NULL_RLP); + assert!(t.is_empty()); + // TODO: make work: + //assert_eq!(t.root(), SHA3_NULL_RLP); +} \ No newline at end of file From cf62f67b5f3d2ab19f054775c9bfe7481aa1e682 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 16:00:23 +0100 Subject: [PATCH 104/379] Fix warnings. --- src/hashdb.rs | 1 - src/memorydb.rs | 7 +++---- src/overlaydb.rs | 1 - src/trie.rs | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/hashdb.rs b/src/hashdb.rs index eb41d23df..561c7843d 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -1,5 +1,4 @@ use hash::*; -use bytes::Bytes; pub trait HashDB { /// Look up a given hash into the bytes that hash to it, returning None if the diff --git a/src/memorydb.rs b/src/memorydb.rs index 58dc980ca..4ec02bd5e 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -104,9 +104,8 @@ impl MemoryDB { pub fn denote(&self, key: &H256, value: Bytes) -> &(Bytes, i32) { if self.data.get(&key) == None { unsafe { - let p = &self.data as *const HashMap; - let mp = p as *mut HashMap; - (*mp).insert(key.clone(), (value, 0)); + let p = &self.data as *const HashMap as *mut HashMap; + (*p).insert(key.clone(), (value, 0)); } } self.data.get(key).unwrap() @@ -161,7 +160,7 @@ fn memorydb_denote() { let hash = m.insert(hello_bytes); assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); - for i in 0..1000 { + for _ in 0..1000 { let r = H256::random(); let k = r.sha3(); let &(ref v, ref rc) = m.denote(&k, r.bytes().to_vec()); diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 86ad266c9..b6f2b3eb2 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -8,7 +8,6 @@ use hashdb::*; use memorydb::*; use std::ops::*; use std::sync::*; -use std::cell::*; use std::env; use rocksdb::{DB, Writable}; diff --git a/src/trie.rs b/src/trie.rs index 41c8f58ea..392e40a04 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,8 +1,8 @@ use memorydb::*; use hashdb::*; use hash::*; -use rlp::*; -use bytes::*; +//use rlp::*; +//use bytes::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); From f7ec667852fc4641ac76d01c70a4bcffc9957182 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 18:44:29 +0100 Subject: [PATCH 105/379] in pro... --- src/bytes.rs | 56 +++---- src/rlp.rs | 90 +++++----- src/triehash.rs | 435 +++++++++++++++++++++++++++++++----------------- src/vector.rs | 37 +++- 4 files changed, 391 insertions(+), 227 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index c08c4ee2a..03815524e 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -104,27 +104,27 @@ impl ToBytes for String { fn to_bytes_len(&self) -> usize { self.len() } } -impl ToBytes for u8 { - fn to_bytes(&self) -> Vec { - match *self { - 0 => vec![], - _ => vec![*self] - } - } +//impl ToBytes for u8 { + //fn to_bytes(&self) -> Vec { + //match *self { + //0 => vec![], + //_ => vec![*self] + //} + //} - fn to_bytes_len(&self) -> usize { - match *self { - 0 => 0, - _ => 1 - } - } - fn first_byte(&self) -> Option { - match *self { - 0 => None, - _ => Some(*self) - } - } -} + //fn to_bytes_len(&self) -> usize { + //match *self { + //0 => 0, + //_ => 1 + //} + //} + //fn first_byte(&self) -> Option { + //match *self { + //0 => None, + //_ => Some(*self) + //} + //} +//} impl ToBytes for u64 { fn to_bytes(&self) -> Vec { @@ -223,14 +223,14 @@ impl FromBytes for String { } } -impl FromBytes for u8 { - fn from_bytes(bytes: &[u8]) -> FromBytesResult { - match bytes.len() { - 0 => Ok(0), - _ => Ok(bytes[0]) - } - } -} +//impl FromBytes for u8 { + //fn from_bytes(bytes: &[u8]) -> FromBytesResult { + //match bytes.len() { + //0 => Ok(0), + //_ => Ok(bytes[0]) + //} + //} +//} impl FromBytes for u64 { fn from_bytes(bytes: &[u8]) -> FromBytesResult { diff --git a/src/rlp.rs b/src/rlp.rs index 5b08aca2b..de31dcb4a 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -478,7 +478,7 @@ impl RlpStream { } /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a RlpStream { + pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { // push raw items self.encoder.bytes.extend(bytes); @@ -745,19 +745,19 @@ mod tests { } } - #[test] - fn encode_u8() { - let tests = vec![ - ETestPair(0u8, vec![0x80u8]), - ETestPair(15, vec![15]), - ETestPair(55, vec![55]), - ETestPair(56, vec![56]), - ETestPair(0x7f, vec![0x7f]), - ETestPair(0x80, vec![0x81, 0x80]), - ETestPair(0xff, vec![0x81, 0xff]), - ]; - run_encode_tests(tests); - } + //#[test] + //fn encode_u8() { + //let tests = vec![ + //ETestPair(0u8, vec![0x80u8]), + //ETestPair(15, vec![15]), + //ETestPair(55, vec![55]), + //ETestPair(56, vec![56]), + //ETestPair(0x7f, vec![0x7f]), + //ETestPair(0x80, vec![0x81, 0x80]), + //ETestPair(0xff, vec![0x81, 0xff]), + //]; + //run_encode_tests(tests); + //} #[test] fn encode_u16() { @@ -835,15 +835,15 @@ mod tests { run_encode_tests(tests); } - #[test] - fn encode_vector_u8() { - let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u8], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_encode_tests(tests); - } + //#[test] + //fn encode_vector_u8() { + //let tests = vec![ + //ETestPair(vec![], vec![0xc0]), + //ETestPair(vec![15u8], vec![0xc1, 0x0f]), + //ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + //]; + //run_encode_tests(tests); + //} #[test] fn encode_vector_u64() { @@ -899,19 +899,19 @@ mod tests { } } - #[test] - fn decode_untrusted_u8() { - let tests = vec![ - DTestPair(0u8, vec![0u8]), - DTestPair(15, vec![15]), - DTestPair(55, vec![55]), - DTestPair(56, vec![56]), - DTestPair(0x7f, vec![0x7f]), - DTestPair(0x80, vec![0x81, 0x80]), - DTestPair(0xff, vec![0x81, 0xff]), - ]; - run_decode_untrusted_tests(tests); - } + //#[test] + //fn decode_untrusted_u8() { + //let tests = vec![ + //DTestPair(0u8, vec![0u8]), + //DTestPair(15, vec![15]), + //DTestPair(55, vec![55]), + //DTestPair(56, vec![56]), + //DTestPair(0x7f, vec![0x7f]), + //DTestPair(0x80, vec![0x81, 0x80]), + //DTestPair(0xff, vec![0x81, 0xff]), + //]; + //run_decode_untrusted_tests(tests); + //} #[test] fn decode_untrusted_u16() { @@ -991,15 +991,15 @@ mod tests { run_decode_untrusted_tests(tests); } - #[test] - fn decode_untrusted_vector_u8() { - let tests = vec![ - DTestPair(vec![] as Vec, vec![0xc0]), - DTestPair(vec![15u8], vec![0xc1, 0x0f]), - DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_decode_untrusted_tests(tests); - } + //#[test] + //fn decode_untrusted_vector_u8() { + //let tests = vec![ + //DTestPair(vec![] as Vec, vec![0xc0]), + //DTestPair(vec![15u8], vec![0xc1, 0x0f]), + //DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + //]; + //run_decode_untrusted_tests(tests); + //} #[test] fn decode_untrusted_vector_u64() { diff --git a/src/triehash.rs b/src/triehash.rs index 7059cb5f9..6f9490cc1 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,171 +1,300 @@ //! Generete trie root -use std::collections::BTreeMap; -use std::cmp; -use hash::*; -use sha3::*; -use rlp; -use rlp::RlpStream; +//use std::collections::BTreeMap; +//use std::cmp; +//use hash::*; +//use sha3::*; +//use rlp; +//use rlp::RlpStream; +//use vector::SharedPrefix; -/// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. -/// -/// The "termination marker" and "leaf-node" specifier are completely equivalent. -/// -/// Input values are in range `[0, 0xf]`. -/// -/// ```markdown -/// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 -/// [0,1,2,3,4,5] 0x00012345 // 6 > 4 -/// [1,2,3,4,5] 0x112345 // 5 > 3 -/// [0,0,1,2,3,4] 0x00001234 // 6 > 3 -/// [0,1,2,3,4] 0x101234 // 5 > 3 -/// [1,2,3,4] 0x001234 // 4 > 3 -/// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 -/// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 -/// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 -/// [1,2,3,4,5,T] 0x312345 // 5 > 3 -/// [1,2,3,4,T] 0x201234 // 4 > 3 -/// ``` -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::triehash::*; -/// -/// fn main() { -/// let v = vec![0, 0, 1, 2, 3, 4, 5]; -/// let e = vec![0x10, 0x01, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, false); -/// assert_eq!(h, e); -/// -/// let v = vec![0, 1, 2, 3, 4, 5]; -/// let e = vec![0x00, 0x01, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, false); -/// assert_eq!(h, e); -/// -/// let v = vec![0, 1, 2, 3, 4, 5]; -/// let e = vec![0x20, 0x01, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, true); -/// assert_eq!(h, e); -/// -/// let v = vec![1, 2, 3, 4, 5]; -/// let e = vec![0x31, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, true); -/// assert_eq!(h, e); -/// -/// let v = vec![1, 2, 3, 4]; -/// let e = vec![0x00, 0x12, 0x34]; -/// let h = hex_prefix_encode(&v, false); -/// assert_eq!(h, e); -/// } -/// ``` -/// -pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { - let inlen = nibbles.len(); - let oddness_factor = inlen % 2; - // next even number divided by two - let reslen = (inlen + 2) >> 1; - let mut res = vec![]; - res.reserve(reslen); +///// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. +///// +///// The "termination marker" and "leaf-node" specifier are completely equivalent. +///// +///// Input values are in range `[0, 0xf]`. +///// +///// ```markdown +///// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 +///// [0,1,2,3,4,5] 0x00012345 // 6 > 4 +///// [1,2,3,4,5] 0x112345 // 5 > 3 +///// [0,0,1,2,3,4] 0x00001234 // 6 > 3 +///// [0,1,2,3,4] 0x101234 // 5 > 3 +///// [1,2,3,4] 0x001234 // 4 > 3 +///// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 +///// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 +///// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 +///// [1,2,3,4,5,T] 0x312345 // 5 > 3 +///// [1,2,3,4,T] 0x201234 // 4 > 3 +///// ``` +///// +///// ```rust +///// extern crate ethcore_util as util; +///// use util::triehash::*; +///// +///// fn main() { +///// let v = vec![0, 0, 1, 2, 3, 4, 5]; +///// let e = vec![0x10, 0x01, 0x23, 0x45]; +///// let h = hex_prefix_encode(&v, false); +///// assert_eq!(h, e); +///// +///// let v = vec![0, 1, 2, 3, 4, 5]; +///// let e = vec![0x00, 0x01, 0x23, 0x45]; +///// let h = hex_prefix_encode(&v, false); +///// assert_eq!(h, e); +///// +///// let v = vec![0, 1, 2, 3, 4, 5]; +///// let e = vec![0x20, 0x01, 0x23, 0x45]; +///// let h = hex_prefix_encode(&v, true); +///// assert_eq!(h, e); +///// +///// let v = vec![1, 2, 3, 4, 5]; +///// let e = vec![0x31, 0x23, 0x45]; +///// let h = hex_prefix_encode(&v, true); +///// assert_eq!(h, e); +///// +///// let v = vec![1, 2, 3, 4]; +///// let e = vec![0x00, 0x12, 0x34]; +///// let h = hex_prefix_encode(&v, false); +///// assert_eq!(h, e); +///// +///// let v = vec![4, 1]; +///// let e = vec![0x20, 0x41]; +///// let h = hex_prefix_encode(&v, true); +///// assert_eq!(h, e); +///// } +///// ``` +///// +//pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { + //let inlen = nibbles.len(); + //let oddness_factor = inlen % 2; + //// next even number divided by two + //let reslen = (inlen + 2) >> 1; + //let mut res = vec![]; + //res.reserve(reslen); - let first_byte = { - let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; - if oddness_factor == 1 { - bits += nibbles[0]; - } - bits - }; + //let first_byte = { + //let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; + //if oddness_factor == 1 { + //bits += nibbles[0]; + //} + //bits + //}; - res.push(first_byte); + //res.push(first_byte); - let mut offset = oddness_factor; - while offset < inlen { - let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; - res.push(byte); - offset += 2; - } + //let mut offset = oddness_factor; + //while offset < inlen { + //let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; + //res.push(byte); + //offset += 2; + //} - res -} + //res +//} -/// Converts slice of bytes to nibbles. -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::triehash::*; -/// -/// fn main () { -/// let v = vec![0x31, 0x23, 0x45]; -/// let e = vec![3, 1, 2, 3, 4, 5]; -/// assert_eq!(as_nibbles(&v), e); -/// } -/// ``` -pub fn as_nibbles(bytes: &[u8]) -> Vec { - let mut res = vec![]; - res.reserve(bytes.len() * 2); - for i in 0..bytes.len() { - res.push(bytes[i] >> 4); - res.push((bytes[i] << 4) >> 4); - } - res -} +///// Converts slice of bytes to nibbles. +///// +///// ```rust +///// extern crate ethcore_util as util; +///// use util::triehash::*; +///// +///// fn main () { +///// let v = vec![0x31, 0x23, 0x45]; +///// let e = vec![3, 1, 2, 3, 4, 5]; +///// assert_eq!(as_nibbles(&v), e); +///// +///// // A => 65 => 0x41 => [4, 1] +///// let v: Vec = From::from("A"); +///// let e = vec![4, 1]; +///// assert_eq!(as_nibbles(&v), e); +///// } +///// ``` +//pub fn as_nibbles(bytes: &[u8]) -> Vec { + //let mut res = vec![]; + //res.reserve(bytes.len() * 2); + //for i in 0..bytes.len() { + //res.push(bytes[i] >> 4); + //res.push((bytes[i] << 4) >> 4); + //} + //res +//} -struct NibblePair { - nibble: Vec, - data: Vec -} +//#[derive(Debug)] +//pub struct NibblePair { + //nibble: Vec, + //data: Vec +//} -pub fn ordered_trie_root(data: Vec>) -> H256 { - let vec: Vec = data - // first put elements into btree to sort them by nibbles - // optimize it later - .into_iter() - .fold(BTreeMap::new(), | mut acc, vec | { - let len = acc.len(); - acc.insert(as_nibbles(&rlp::encode(&len)), vec); - acc - }) - // then move them to a vector - .into_iter() - .map(|(k, v)| NibblePair { nibble: k, data: v } ) - .collect(); +//impl NibblePair { + //pub fn new(nibble: Vec, data: Vec) -> NibblePair { + //NibblePair { + //nibble: nibble, + //data: data + //} + //} - let out = match vec.len() { - 0 => rlp::encode(&""), - _ => { - let mut stream = RlpStream::new(); - hash256rlp(&vec, 0, &mut stream); - stream.out().unwrap() - } - }; + //pub fn new_raw(to_nibble: Vec, data: Vec) -> NibblePair { + //NibblePair::new(as_nibbles(&to_nibble), data) + //} +//} + +//pub fn ordered_trie_root(data: Vec>) -> H256 { + //let vec: Vec = data + //// first put elements into btree to sort them by nibbles + //// optimize it later + //.into_iter() + //.fold(BTreeMap::new(), | mut acc, vec | { + //let len = acc.len(); + //acc.insert(as_nibbles(&rlp::encode(&len)), vec); + //acc + //}) + //// then move them to a vector + //.into_iter() + //.map(|(k, v)| NibblePair::new(k, v) ) + //.collect(); + + //hash256(&vec) +//} + +//pub fn hash256(vec: &[NibblePair]) -> H256 { + //let out = match vec.len() { + //0 => rlp::encode(&""), + //_ => { + //let mut stream = RlpStream::new(); + //hash256rlp(&vec, 0, &mut stream); + //stream.out().unwrap() + //} + //}; - out.sha3() -} + //println!("out: {:?}", out); + //out.sha3() +//} -fn shared_prefix_length(v1: &[T], v2: &[T]) -> usize where T: Eq { - let len = cmp::min(v1.len(), v2.len()); - (0..len).take_while(|&i| v1[i] == v2[i]).count() -} +//fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { + //match vec.len() { + //0 => { + //stream.append(&""); + //}, + //1 => { + //stream.append_list(2); + //stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..], true)); + //stream.append(&vec[0].data); + //}, + //_ => { + //let shared_prefix = vec.iter() + //// skip first element + //.skip(1) + //// get minimum number of shared nibbles between first and each successive + //.fold(usize::max_value(), | acc, pair | { + //cmp::min(vec[0].nibble.shared_prefix_len(&pair.nibble), acc) + //}); -fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { - match vec.len() { - 0 => stream.append(&""), - 1 => stream.append_list(2).append(&hex_prefix_encode(&vec[0].nibble, true)).append(&vec[0].data), - _ => { - let shared_prefix = vec.iter() - // skip first element - .skip(1) - // get minimum number of shared nibbles between first string and each successive - .fold(usize::max_value(), | acc, pair | cmp::min(shared_prefix_length(&vec[0].nibble, &pair.nibble), acc) ); //match shared_prefix > pre_len { + //true => { + //stream.append_list(2); + //stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..shared_prefix], false)); + //hash256aux(vec, shared_prefix, stream); + //}, + //false => { + //stream.append_list(17); - //true => hex_prefix_encode(&vec[0].nibble + //// every nibble is longer then previous one + //let iter = vec.iter() + //// move to first element with len different then pre_len + //.take_while(| pair | { pair.nibble.len() == pre_len }); + + //let mut begin = iter.count(); + + //for i in 0..16 { + //// cout how many successive elements have same next nibble + //let len = vec[begin..].iter() + //.map(| pair | pair.nibble[pre_len] ) + //.take_while(|&q| q == i).count(); + + //match len { + //0 => { stream.append(&""); }, + //_ => hash256aux(&vec[begin..begin + len], pre_len + 1, stream) + //} + //begin += len; + //} + + //match pre_len == vec[0].nibble.len() { + //true => stream.append(&vec[0].data), + //false => stream.append(&"") + //}; + //} //} - panic!(); - - } - }; -} + //} + //}; +//} -#[cfg(test)] -mod tests { -} +//fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { + //let mut s = RlpStream::new(); + //hash256rlp(vec, pre_len, &mut s); + //let out = s.out().unwrap(); + //match out.len() { + //0...31 => stream.append_raw(&out, 1), + //_ => stream.append(&out.sha3()) + //}; +//} + +//#[cfg(test)] +//mod tests { + //use std::str::FromStr; + //use rustc_serialize::hex::FromHex; + //use hash::*; + //use triehash::*; + + //#[test] + //fn empty_trie_root() { + //assert_eq!(hash256(&vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); + //} + + //#[test] + //fn single_trie_item() { + + //let v = vec![ + //NibblePair::new_raw(From::from("A"), + //From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + //]; + //println!("{:?}", v); + //assert_eq!(hash256(&v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); + //} + + //#[test] + //fn test_trie_root() { + //let v = vec![ + //NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + //"22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), + + //NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), + //"67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + //NibblePair::new_raw("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), + //"6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), + + //NibblePair::new_raw("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), + //"4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + //NibblePair::new_raw("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), + //"4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + //NibblePair::new_raw("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), + //"7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), + + //NibblePair::new_raw("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), + //"ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + + //NibblePair::new_raw("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), + //"697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) + //]; + + //let root = H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap(); + + //let res = hash256(&v); + + //println!("{:?}", res); + ////assert!(false); + //} +//} diff --git a/src/vector.rs b/src/vector.rs index 7032335cc..41c932c92 100644 --- a/src/vector.rs +++ b/src/vector.rs @@ -1,3 +1,5 @@ +//! vector util functions + use std::ptr; pub trait InsertSlice { @@ -32,5 +34,38 @@ impl InsertSlice for Vec { } } -pub trait SharedPreifx { +/// Returns len of prefix shared with elem +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::vector::SharedPrefix; +/// +/// fn main () { +/// let a = vec![1,2,3,3,5]; +/// let b = vec![1,2,3]; +/// assert_eq!(a.shared_prefix_len(&b), 3); +/// } +/// ``` +pub trait SharedPrefix { + fn shared_prefix_len(&self, elem: &[T]) -> usize; +} + +impl SharedPrefix for Vec where T: Eq { + fn shared_prefix_len(&self, elem: &[T]) -> usize { + use std::cmp; + let len = cmp::min(self.len(), elem.len()); + (0..len).take_while(|&i| self[i] == elem[i]).count() + } +} + +#[cfg(test)] +mod test { + use vector::SharedPrefix; + + #[test] + fn test_shared_prefix() { + let a = vec![1,2,3,3,5]; + let b = vec![1,2,3]; + assert_eq!(a.shared_prefix_len(&b), 3); + } } From 9c632d58f2d1b6270087e50e58a9240254adf1f6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 18:45:41 +0100 Subject: [PATCH 106/379] Possible fix? --- src/trie.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 392e40a04..889c9b5dc 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -17,10 +17,10 @@ pub trait Trie { fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } // TODO: consider returning &[u8]... -/* fn at(key: &[u8]) -> Option; + fn contains(key: &[u8]) -> bool; + fn at(key: &[u8]) -> Option<&[u8]>; fn insert(key: &[u8], value: &[u8]); fn remove(key: &[u8]); - fn contains(key: &[u8]) -> bool;*/ } pub struct TrieDB { @@ -39,14 +39,28 @@ impl TrieDB { pub fn db(&self) -> &HashDB { self.db.as_ref() } -// pub fn db_mut(&mut self) -> &mut HashDB { self.db.as } - fn insert_root(&mut self, root_data: &[u8]) { self.root = self.db.insert(root_data); } } impl Trie for TrieDB { fn root(&self) -> &H256 { &self.root } + + fn contains(_key: &[u8]) -> bool { + unimplemented!(); + } + + fn at(_key: &[u8]) -> Option<&[u8]> { + unimplemented!(); + } + + fn insert(_key: &[u8], _value: &[u8]) { + unimplemented!(); + } + + fn remove(_key: &[u8]) { + unimplemented!(); + } } #[test] From 03c1944d9c422833a2ee7e59ac0879f810b2964c Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 19:35:22 +0100 Subject: [PATCH 107/379] fixed encoding of byte vectors --- src/bytes.rs | 34 +--------- src/rlp.rs | 179 ++++++++++++++++++++++++--------------------------- 2 files changed, 86 insertions(+), 127 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 03815524e..e17ccca9a 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -23,7 +23,7 @@ //! use util::bytes::FromBytes; //! //! let a = String::from_bytes(&[b'd', b'o', b'g']); -//! let b = u8::from_bytes(&[0xfa]); +//! let b = u16::from_bytes(&[0xfa]); //! let c = u64::from_bytes(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); //! } //! @@ -104,28 +104,6 @@ impl ToBytes for String { fn to_bytes_len(&self) -> usize { self.len() } } -//impl ToBytes for u8 { - //fn to_bytes(&self) -> Vec { - //match *self { - //0 => vec![], - //_ => vec![*self] - //} - //} - - //fn to_bytes_len(&self) -> usize { - //match *self { - //0 => 0, - //_ => 1 - //} - //} - //fn first_byte(&self) -> Option { - //match *self { - //0 => None, - //_ => Some(*self) - //} - //} -//} - impl ToBytes for u64 { fn to_bytes(&self) -> Vec { let mut res= vec![]; @@ -223,15 +201,6 @@ impl FromBytes for String { } } -//impl FromBytes for u8 { - //fn from_bytes(bytes: &[u8]) -> FromBytesResult { - //match bytes.len() { - //0 => Ok(0), - //_ => Ok(bytes[0]) - //} - //} -//} - impl FromBytes for u64 { fn from_bytes(bytes: &[u8]) -> FromBytesResult { match bytes.len() { @@ -293,3 +262,4 @@ impl FromBytes for T where T: FixedHash { } } } + diff --git a/src/rlp.rs b/src/rlp.rs index de31dcb4a..feae875e3 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -61,11 +61,11 @@ //! // [ [], [[]], [ [], [[]] ] ] //! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; //! let rlp = Rlp::new(&data); -//! let _v0: Vec = Decodable::decode(&rlp.at(0)); -//! let _v1: Vec> = Decodable::decode(&rlp.at(1)); +//! let _v0: Vec = Decodable::decode(&rlp.at(0)); +//! let _v1: Vec> = Decodable::decode(&rlp.at(1)); //! let nested_rlp = rlp.at(2); -//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); -//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); +//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); +//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); //! } //! //! fn main() { @@ -343,9 +343,7 @@ pub fn decode(bytes: &[u8]) -> T where T: Decodable { } /// shortcut function to decode UntrustedRlp `&[u8]` into an object -pub fn decode_untrusted(bytes: &[u8]) -> Result - where T: Decodable -{ +pub fn decode_untrusted(bytes: &[u8]) -> Result where T: Decodable { let rlp = UntrustedRlp::new(bytes); T::decode_untrusted(&rlp) } @@ -357,18 +355,18 @@ pub trait Decodable: Sized { } } -impl Decodable for T where T: FromBytes -{ +impl Decodable for T where T: FromBytes { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { match rlp.is_value() { - true => BasicDecoder::read_value(rlp.bytes), + true => BasicDecoder::read_value(rlp.bytes, | bytes | { + Ok(try!(T::from_bytes(bytes))) + }), false => Err(DecoderError::UntrustedRlpExpectedToBeValue), } } } -impl Decodable for Vec where T: Decodable -{ +impl Decodable for Vec where T: Decodable { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { match rlp.is_list() { true => rlp.iter().map(|rlp| T::decode_untrusted(&rlp)).collect(), @@ -377,29 +375,40 @@ impl Decodable for Vec where T: Decodable } } +impl Decodable for Vec { + fn decode_untrusted(rlp: &UntrustedRlp) -> Result { + match rlp.is_value() { + true => BasicDecoder::read_value(rlp.bytes, | bytes | { + let mut res = vec![]; + res.extend(bytes); + Ok(res) + }), + false => Err(DecoderError::UntrustedRlpExpectedToBeValue), + } + } +} + pub trait Decoder { - fn read_value(bytes: &[u8]) -> Result where T: FromBytes; + fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result; } struct BasicDecoder; impl Decoder for BasicDecoder { - fn read_value(bytes: &[u8]) -> Result - where T: FromBytes - { + fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { match bytes.first().map(|&x| x) { // rlp is too short None => Err(DecoderError::UntrustedRlpIsTooShort), // single byt value - Some(l @ 0...0x7f) => Ok(try!(T::from_bytes(&[l]))), + Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), // 0-55 bytes - Some(l @ 0x80...0xb7) => Ok(try!(T::from_bytes(&bytes[1..(1 + l as usize - 0x80)]))), + Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), // longer than 55 bytes Some(l @ 0xb8...0xbf) => { let len_of_len = l as usize - 0xb7; let begin_of_value = 1 as usize + len_of_len; let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); - Ok(try!(T::from_bytes(&bytes[begin_of_value..begin_of_value + len]))) + Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } _ => Err(DecoderError::BadUntrustedRlp), } @@ -525,8 +534,7 @@ impl RlpStream { } /// shortcut function to encode a `T: Encodable` into a UntrustedRlp `Vec` -pub fn encode(object: &E) -> Vec - where E: Encodable +pub fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); object.encode(&mut encoder); @@ -559,20 +567,14 @@ pub trait Encoder { fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); } -impl Encodable for T where T: ToBytes -{ - fn encode(&self, encoder: &mut E) -> () - where E: Encoder - { +impl Encodable for T where T: ToBytes { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_value(&self.to_bytes()) } } -impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a -{ - fn encode(&self, encoder: &mut E) -> () - where E: Encoder - { +impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_list(|e| { // insert all list elements for el in self.iter() { @@ -582,16 +584,29 @@ impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a } } -impl Encodable for Vec where T: Encodable -{ - fn encode(&self, encoder: &mut E) -> () - where E: Encoder - { +impl Encodable for Vec where T: Encodable { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { let r: &[T] = self.as_ref(); r.encode(encoder) } } +/// lets treat bytes differently than other lists +/// they are a single value +impl<'a> Encodable for &'a [u8] { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} + +/// lets treat bytes differently than other lists +/// they are a single value +impl Encodable for Vec { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} + struct BasicEncoder { bytes: Vec, } @@ -744,21 +759,7 @@ mod tests { assert_eq!(res, &t.1[..]); } } - - //#[test] - //fn encode_u8() { - //let tests = vec![ - //ETestPair(0u8, vec![0x80u8]), - //ETestPair(15, vec![15]), - //ETestPair(55, vec![55]), - //ETestPair(56, vec![56]), - //ETestPair(0x7f, vec![0x7f]), - //ETestPair(0x80, vec![0x81, 0x80]), - //ETestPair(0xff, vec![0x81, 0xff]), - //]; - //run_encode_tests(tests); - //} - + #[test] fn encode_u16() { let tests = vec![ @@ -835,15 +836,17 @@ mod tests { run_encode_tests(tests); } - //#[test] - //fn encode_vector_u8() { - //let tests = vec![ - //ETestPair(vec![], vec![0xc0]), - //ETestPair(vec![15u8], vec![0xc1, 0x0f]), - //ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - //]; - //run_encode_tests(tests); - //} + /// Vec is treated as a single value + #[test] + fn encode_vector_u8() { + let tests = vec![ + ETestPair(vec![], vec![0x80]), + ETestPair(vec![0u8], vec![0]), + ETestPair(vec![0x15], vec![0x15]), + ETestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_encode_tests(tests); + } #[test] fn encode_vector_u64() { @@ -890,28 +893,24 @@ mod tests { struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; - fn run_decode_untrusted_tests(tests: Vec>) - where T: rlp::Decodable + fmt::Debug + cmp::Eq - { + fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { for t in &tests { let res: T = rlp::decode_untrusted(&t.1).unwrap(); assert_eq!(res, t.0); } } - //#[test] - //fn decode_untrusted_u8() { - //let tests = vec![ - //DTestPair(0u8, vec![0u8]), - //DTestPair(15, vec![15]), - //DTestPair(55, vec![55]), - //DTestPair(56, vec![56]), - //DTestPair(0x7f, vec![0x7f]), - //DTestPair(0x80, vec![0x81, 0x80]), - //DTestPair(0xff, vec![0x81, 0xff]), - //]; - //run_decode_untrusted_tests(tests); - //} + /// Vec is treated as a single value + #[test] + fn decode_vector_u8() { + let tests = vec![ + DTestPair(vec![], vec![0x80]), + DTestPair(vec![0u8], vec![0]), + DTestPair(vec![0x15], vec![0x15]), + DTestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_decode_tests(tests); + } #[test] fn decode_untrusted_u16() { @@ -920,7 +919,7 @@ mod tests { DTestPair(0x100, vec![0x82, 0x01, 0x00]), DTestPair(0xffff, vec![0x82, 0xff, 0xff]), ]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] @@ -930,7 +929,7 @@ mod tests { DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), ]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] @@ -940,7 +939,7 @@ mod tests { DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), ]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] @@ -956,7 +955,7 @@ mod tests { 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] @@ -975,7 +974,7 @@ mod tests { b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', b'e', b'l', b'i', b't'])]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] @@ -988,19 +987,9 @@ mod tests { 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, 0xb3, 0x7d, 0x11, 0x06]) ]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } - //#[test] - //fn decode_untrusted_vector_u8() { - //let tests = vec![ - //DTestPair(vec![] as Vec, vec![0xc0]), - //DTestPair(vec![15u8], vec![0xc1, 0x0f]), - //DTestPair(vec![1u8, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - //]; - //run_decode_untrusted_tests(tests); - //} - #[test] fn decode_untrusted_vector_u64() { let tests = vec![ @@ -1009,20 +998,20 @@ mod tests { DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), ]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] fn decode_untrusted_vector_str() { let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } #[test] fn decode_untrusted_vector_of_vectors_str() { let tests = vec![DTestPair(vec![vec!["cat".to_string()]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_decode_untrusted_tests(tests); + run_decode_tests(tests); } } From f4292134c0872274e34578707f3a3ba5cb9aac1f Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 29 Nov 2015 19:42:03 +0100 Subject: [PATCH 108/379] uncommented triehash --- src/triehash.rs | 521 ++++++++++++++++++++++++------------------------ 1 file changed, 260 insertions(+), 261 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 6f9490cc1..ad553cf70 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,300 +1,299 @@ //! Generete trie root -//use std::collections::BTreeMap; -//use std::cmp; -//use hash::*; -//use sha3::*; -//use rlp; -//use rlp::RlpStream; -//use vector::SharedPrefix; +use std::collections::BTreeMap; +use std::cmp; +use hash::*; +use sha3::*; +use rlp; +use rlp::RlpStream; +use vector::SharedPrefix; -///// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. -///// -///// The "termination marker" and "leaf-node" specifier are completely equivalent. -///// -///// Input values are in range `[0, 0xf]`. -///// -///// ```markdown -///// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 -///// [0,1,2,3,4,5] 0x00012345 // 6 > 4 -///// [1,2,3,4,5] 0x112345 // 5 > 3 -///// [0,0,1,2,3,4] 0x00001234 // 6 > 3 -///// [0,1,2,3,4] 0x101234 // 5 > 3 -///// [1,2,3,4] 0x001234 // 4 > 3 -///// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 -///// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 -///// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 -///// [1,2,3,4,5,T] 0x312345 // 5 > 3 -///// [1,2,3,4,T] 0x201234 // 4 > 3 -///// ``` -///// -///// ```rust -///// extern crate ethcore_util as util; -///// use util::triehash::*; -///// -///// fn main() { -///// let v = vec![0, 0, 1, 2, 3, 4, 5]; -///// let e = vec![0x10, 0x01, 0x23, 0x45]; -///// let h = hex_prefix_encode(&v, false); -///// assert_eq!(h, e); -///// -///// let v = vec![0, 1, 2, 3, 4, 5]; -///// let e = vec![0x00, 0x01, 0x23, 0x45]; -///// let h = hex_prefix_encode(&v, false); -///// assert_eq!(h, e); -///// -///// let v = vec![0, 1, 2, 3, 4, 5]; -///// let e = vec![0x20, 0x01, 0x23, 0x45]; -///// let h = hex_prefix_encode(&v, true); -///// assert_eq!(h, e); -///// -///// let v = vec![1, 2, 3, 4, 5]; -///// let e = vec![0x31, 0x23, 0x45]; -///// let h = hex_prefix_encode(&v, true); -///// assert_eq!(h, e); -///// -///// let v = vec![1, 2, 3, 4]; -///// let e = vec![0x00, 0x12, 0x34]; -///// let h = hex_prefix_encode(&v, false); -///// assert_eq!(h, e); -///// -///// let v = vec![4, 1]; -///// let e = vec![0x20, 0x41]; -///// let h = hex_prefix_encode(&v, true); -///// assert_eq!(h, e); -///// } -///// ``` -///// -//pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { - //let inlen = nibbles.len(); - //let oddness_factor = inlen % 2; - //// next even number divided by two - //let reslen = (inlen + 2) >> 1; - //let mut res = vec![]; - //res.reserve(reslen); +/// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. +/// +/// The "termination marker" and "leaf-node" specifier are completely equivalent. +/// +/// Input values are in range `[0, 0xf]`. +/// +/// ```markdown +/// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 +/// [0,1,2,3,4,5] 0x00012345 // 6 > 4 +/// [1,2,3,4,5] 0x112345 // 5 > 3 +/// [0,0,1,2,3,4] 0x00001234 // 6 > 3 +/// [0,1,2,3,4] 0x101234 // 5 > 3 +/// [1,2,3,4] 0x001234 // 4 > 3 +/// [0,0,1,2,3,4,5,T] 0x30012345 // 7 > 4 +/// [0,0,1,2,3,4,T] 0x20001234 // 6 > 4 +/// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 +/// [1,2,3,4,5,T] 0x312345 // 5 > 3 +/// [1,2,3,4,T] 0x201234 // 4 > 3 +/// ``` +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::triehash::*; +/// +/// fn main() { +/// let v = vec![0, 0, 1, 2, 3, 4, 5]; +/// let e = vec![0x10, 0x01, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, false); +/// assert_eq!(h, e); +/// +/// let v = vec![0, 1, 2, 3, 4, 5]; +/// let e = vec![0x00, 0x01, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, false); +/// assert_eq!(h, e); +/// +/// let v = vec![0, 1, 2, 3, 4, 5]; +/// let e = vec![0x20, 0x01, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, true); +/// assert_eq!(h, e); +/// +/// let v = vec![1, 2, 3, 4, 5]; +/// let e = vec![0x31, 0x23, 0x45]; +/// let h = hex_prefix_encode(&v, true); +/// assert_eq!(h, e); +/// +/// let v = vec![1, 2, 3, 4]; +/// let e = vec![0x00, 0x12, 0x34]; +/// let h = hex_prefix_encode(&v, false); +/// assert_eq!(h, e); +/// +/// let v = vec![4, 1]; +/// let e = vec![0x20, 0x41]; +/// let h = hex_prefix_encode(&v, true); +/// assert_eq!(h, e); +/// } +/// ``` +/// +pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { + let inlen = nibbles.len(); + let oddness_factor = inlen % 2; + // next even number divided by two + let reslen = (inlen + 2) >> 1; + let mut res = vec![]; + res.reserve(reslen); - //let first_byte = { - //let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; - //if oddness_factor == 1 { - //bits += nibbles[0]; - //} - //bits - //}; + let first_byte = { + let mut bits = ((inlen as u8 & 1) + (2 * leaf as u8)) << 4; + if oddness_factor == 1 { + bits += nibbles[0]; + } + bits + }; - //res.push(first_byte); + res.push(first_byte); - //let mut offset = oddness_factor; - //while offset < inlen { - //let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; - //res.push(byte); - //offset += 2; - //} + let mut offset = oddness_factor; + while offset < inlen { + let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; + res.push(byte); + offset += 2; + } - //res -//} + res +} -///// Converts slice of bytes to nibbles. -///// -///// ```rust -///// extern crate ethcore_util as util; -///// use util::triehash::*; -///// -///// fn main () { -///// let v = vec![0x31, 0x23, 0x45]; -///// let e = vec![3, 1, 2, 3, 4, 5]; -///// assert_eq!(as_nibbles(&v), e); -///// -///// // A => 65 => 0x41 => [4, 1] -///// let v: Vec = From::from("A"); -///// let e = vec![4, 1]; -///// assert_eq!(as_nibbles(&v), e); -///// } -///// ``` -//pub fn as_nibbles(bytes: &[u8]) -> Vec { - //let mut res = vec![]; - //res.reserve(bytes.len() * 2); - //for i in 0..bytes.len() { - //res.push(bytes[i] >> 4); - //res.push((bytes[i] << 4) >> 4); - //} - //res -//} +/// Converts slice of bytes to nibbles. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::triehash::*; +/// +/// fn main () { +/// let v = vec![0x31, 0x23, 0x45]; +/// let e = vec![3, 1, 2, 3, 4, 5]; +/// assert_eq!(as_nibbles(&v), e); +/// +/// // A => 65 => 0x41 => [4, 1] +/// let v: Vec = From::from("A"); +/// let e = vec![4, 1]; +/// assert_eq!(as_nibbles(&v), e); +/// } +/// ``` +pub fn as_nibbles(bytes: &[u8]) -> Vec { + let mut res = vec![]; + res.reserve(bytes.len() * 2); + for i in 0..bytes.len() { + res.push(bytes[i] >> 4); + res.push((bytes[i] << 4) >> 4); + } + res +} -//#[derive(Debug)] -//pub struct NibblePair { - //nibble: Vec, - //data: Vec -//} +#[derive(Debug)] +pub struct NibblePair { + nibble: Vec, + data: Vec +} -//impl NibblePair { - //pub fn new(nibble: Vec, data: Vec) -> NibblePair { - //NibblePair { - //nibble: nibble, - //data: data - //} - //} +impl NibblePair { + pub fn new(nibble: Vec, data: Vec) -> NibblePair { + NibblePair { + nibble: nibble, + data: data + } + } - //pub fn new_raw(to_nibble: Vec, data: Vec) -> NibblePair { - //NibblePair::new(as_nibbles(&to_nibble), data) - //} -//} + pub fn new_raw(to_nibble: Vec, data: Vec) -> NibblePair { + NibblePair::new(as_nibbles(&to_nibble), data) + } +} -//pub fn ordered_trie_root(data: Vec>) -> H256 { - //let vec: Vec = data - //// first put elements into btree to sort them by nibbles - //// optimize it later - //.into_iter() - //.fold(BTreeMap::new(), | mut acc, vec | { - //let len = acc.len(); - //acc.insert(as_nibbles(&rlp::encode(&len)), vec); - //acc - //}) - //// then move them to a vector - //.into_iter() - //.map(|(k, v)| NibblePair::new(k, v) ) - //.collect(); +pub fn ordered_trie_root(data: Vec>) -> H256 { + let vec: Vec = data + // first put elements into btree to sort them by nibbles + // optimize it later + .into_iter() + .fold(BTreeMap::new(), | mut acc, vec | { + let len = acc.len(); + acc.insert(as_nibbles(&rlp::encode(&len)), vec); + acc + }) + // then move them to a vector + .into_iter() + .map(|(k, v)| NibblePair::new(k, v) ) + .collect(); - //hash256(&vec) -//} + hash256(&vec) +} -//pub fn hash256(vec: &[NibblePair]) -> H256 { - //let out = match vec.len() { - //0 => rlp::encode(&""), - //_ => { - //let mut stream = RlpStream::new(); - //hash256rlp(&vec, 0, &mut stream); - //stream.out().unwrap() - //} - //}; +pub fn hash256(vec: &[NibblePair]) -> H256 { + let out = match vec.len() { + 0 => rlp::encode(&""), + _ => { + let mut stream = RlpStream::new(); + hash256rlp(&vec, 0, &mut stream); + stream.out().unwrap() + } + }; - //println!("out: {:?}", out); - //out.sha3() -//} + println!("out: {:?}", out); + out.sha3() +} -//fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { - //match vec.len() { - //0 => { - //stream.append(&""); - //}, - //1 => { - //stream.append_list(2); - //stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..], true)); - //stream.append(&vec[0].data); - //}, - //_ => { - //let shared_prefix = vec.iter() - //// skip first element - //.skip(1) - //// get minimum number of shared nibbles between first and each successive - //.fold(usize::max_value(), | acc, pair | { - //cmp::min(vec[0].nibble.shared_prefix_len(&pair.nibble), acc) - //}); +fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { + match vec.len() { + 0 => { + stream.append(&""); + }, + 1 => { + stream.append_list(2); + stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..], true)); + stream.append(&vec[0].data); + }, + _ => { + let shared_prefix = vec.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first and each successive + .fold(usize::max_value(), | acc, pair | { + cmp::min(vec[0].nibble.shared_prefix_len(&pair.nibble), acc) + }); - //match shared_prefix > pre_len { - //true => { - //stream.append_list(2); - //stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..shared_prefix], false)); - //hash256aux(vec, shared_prefix, stream); - //}, - //false => { - //stream.append_list(17); + match shared_prefix > pre_len { + true => { + stream.append_list(2); + stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..shared_prefix], false)); + hash256aux(vec, shared_prefix, stream); + }, + false => { + stream.append_list(17); - //// every nibble is longer then previous one - //let iter = vec.iter() - //// move to first element with len different then pre_len - //.take_while(| pair | { pair.nibble.len() == pre_len }); + // every nibble is longer then previous one + let iter = vec.iter() + // move to first element with len different then pre_len + .take_while(| pair | { pair.nibble.len() == pre_len }); - //let mut begin = iter.count(); + let mut begin = iter.count(); - //for i in 0..16 { - //// cout how many successive elements have same next nibble - //let len = vec[begin..].iter() - //.map(| pair | pair.nibble[pre_len] ) - //.take_while(|&q| q == i).count(); + for i in 0..16 { + // cout how many successive elements have same next nibble + let len = vec[begin..].iter() + .map(| pair | pair.nibble[pre_len] ) + .take_while(|&q| q == i).count(); - //match len { - //0 => { stream.append(&""); }, - //_ => hash256aux(&vec[begin..begin + len], pre_len + 1, stream) - //} - //begin += len; - //} + match len { + 0 => { stream.append(&""); }, + _ => hash256aux(&vec[begin..begin + len], pre_len + 1, stream) + } + begin += len; + } - //match pre_len == vec[0].nibble.len() { - //true => stream.append(&vec[0].data), - //false => stream.append(&"") - //}; - //} - //} - //} - //}; -//} + match pre_len == vec[0].nibble.len() { + true => stream.append(&vec[0].data), + false => stream.append(&"") + }; + } + } + } + }; +} -//fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { - //let mut s = RlpStream::new(); - //hash256rlp(vec, pre_len, &mut s); - //let out = s.out().unwrap(); - //match out.len() { - //0...31 => stream.append_raw(&out, 1), - //_ => stream.append(&out.sha3()) - //}; -//} +fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { + let mut s = RlpStream::new(); + hash256rlp(vec, pre_len, &mut s); + let out = s.out().unwrap(); + match out.len() { + 0...31 => stream.append_raw(&out, 1), + _ => stream.append(&out.sha3()) + }; +} -//#[cfg(test)] -//mod tests { - //use std::str::FromStr; - //use rustc_serialize::hex::FromHex; - //use hash::*; - //use triehash::*; +#[cfg(test)] +mod tests { + use std::str::FromStr; + use rustc_serialize::hex::FromHex; + use hash::*; + use triehash::*; - //#[test] - //fn empty_trie_root() { - //assert_eq!(hash256(&vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); - //} + #[test] + fn empty_trie_root() { + assert_eq!(hash256(&vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); + } - //#[test] - //fn single_trie_item() { + #[test] + fn single_trie_item() { - //let v = vec![ - //NibblePair::new_raw(From::from("A"), - //From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) - //]; - //println!("{:?}", v); - //assert_eq!(hash256(&v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); - //} + let v = vec![ + NibblePair::new_raw(From::from("A"), + From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) + ]; + + assert_eq!(hash256(&v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); + } - //#[test] - //fn test_trie_root() { - //let v = vec![ - //NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), - //"22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), + #[test] + fn test_trie_root() { + let _v = vec![ + NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), - //NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), - //"67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), + "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - //NibblePair::new_raw("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), - //"6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), + NibblePair::new_raw("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), + "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), - //NibblePair::new_raw("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), - //"4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + NibblePair::new_raw("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), + "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - //NibblePair::new_raw("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), - //"4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), + NibblePair::new_raw("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), + "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), - //NibblePair::new_raw("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), - //"7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), + NibblePair::new_raw("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), - //NibblePair::new_raw("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), - //"ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + NibblePair::new_raw("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), - //NibblePair::new_raw("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), - //"697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) - //]; + NibblePair::new_raw("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), + "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) + ]; - //let root = H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap(); + let _root = H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap(); //let res = hash256(&v); - //println!("{:?}", res); - ////assert!(false); - //} -//} + //assert_eq!(res, root); + } +} From 06bbefd07b5c511ae00116abf1e5610dc73aa060 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 20:11:02 +0100 Subject: [PATCH 109/379] Nibbleslice basic operations. --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 983dc322c..e2cb3a754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,6 +35,7 @@ pub mod chainfilter; pub mod crypto; pub mod triehash; pub mod trie; +pub mod nibbleslice; //pub mod network; From 930865fefb69c6f8285e1f745c06d4adf8577fa6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 21:17:07 +0100 Subject: [PATCH 110/379] NibbleSlice struct. --- src/nibbleslice.rs | 138 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 src/nibbleslice.rs diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs new file mode 100644 index 000000000..35068f0a2 --- /dev/null +++ b/src/nibbleslice.rs @@ -0,0 +1,138 @@ +use std::cmp::*; + +/// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. +/// +/// This is an immutable struct. No operations actually change it. +#[derive(Debug, Copy, Clone, Eq, Ord)] +pub struct NibbleSlice<'a> { + data: &'a [u8], + offset: usize, +} + +impl<'a> NibbleSlice<'a> { + /// Create a new nibble slice with the given byte-slice. + pub fn new(data: &[u8]) -> NibbleSlice { NibbleSlice::new_offset(data, 0) } + + /// Create a new nibble slice with the given byte-slice with a nibble offset. + pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset} } + + /// Is this an empty slice? + pub fn is_empty(&self) -> bool { self.len() == 0 } + + /// Get the length (in nibbles, naturally) of this slice. + pub fn len(&self) -> usize { self.data.len() * 2 - self.offset } + + /// Get the nibble at position `i`. + pub fn at(&self, i: usize) -> u8 { + if (self.offset + i) & 1 == 1 { + self.data[(self.offset + i) / 2] & 15u8 + } + else { + self.data[(self.offset + i) / 2] >> 4 + } + } + + /// Return object which represents a view on to this slice (further) offset by `i` nibbles. + pub fn mid(&self, i: usize) -> Self { NibbleSlice{ data: self.data, offset: self.offset + i} } + + /// Do we start with the same nibbles as the whole of `them`? + pub fn starts_with(&self, them: &Self) -> bool { self.common_prefix(them) == them.len() } + + /// How many of the same nibbles at the beginning do we match with `them`? + pub fn common_prefix(&self, them: &Self) -> usize { + let s = min(self.len(), them.len()); + let mut i = 0usize; + while i < s { + if self.at(i) != them.at(i) { break; } + i += 1; + } + i + } +} + +impl<'a> PartialEq for NibbleSlice<'a> { + fn eq(&self, them: &Self) -> bool { + self.len() == them.len() && self.starts_with(them) + } +} + +impl<'a> PartialOrd for NibbleSlice<'a> { + fn partial_cmp(&self, them: &Self) -> Option { + let s = min(self.len(), them.len()); + let mut i = 0usize; + while i < s { + match self.at(i).partial_cmp(&them.at(i)).unwrap() { + Ordering::Less => return Some(Ordering::Less), + Ordering::Greater => return Some(Ordering::Greater), + _ => i += 1, + } + } + self.len().partial_cmp(&them.len()) + } +} + +#[cfg(test)] +mod tests { + use super::NibbleSlice; + static D: &'static [u8;3] = &[0x01u8, 0x23, 0x45]; + + #[test] + fn basics() { + let n = NibbleSlice::new(D); + assert_eq!(n.len(), 6); + assert!(!n.is_empty()); + + let n = NibbleSlice::new_offset(D, 6); + assert!(n.is_empty()); + + let n = NibbleSlice::new_offset(D, 3); + assert_eq!(n.len(), 3); + for i in 0..3 { + assert_eq!(n.at(i), i as u8 + 3); + } + } + + #[test] + fn mid() { + let n = NibbleSlice::new(D); + let m = n.mid(2); + for i in 0..4 { + assert_eq!(m.at(i), i as u8 + 2); + } + let m = n.mid(3); + for i in 0..3 { + assert_eq!(m.at(i), i as u8 + 3); + } + } + + #[test] + fn shared() { + let n = NibbleSlice::new(D); + + let other = &[0x01u8, 0x23, 0x01, 0x23, 0x45, 0x67]; + let m = NibbleSlice::new(other); + + assert_eq!(n.common_prefix(&m), 4); + assert_eq!(m.common_prefix(&n), 4); + assert_eq!(n.mid(1).common_prefix(&m.mid(1)), 3); + assert_eq!(n.mid(1).common_prefix(&m.mid(2)), 0); + assert_eq!(n.common_prefix(&m.mid(4)), 6); + assert!(!n.starts_with(&m.mid(4))); + assert!(m.mid(4).starts_with(&n)); + } + + #[test] + fn compare() { + let other = &[0x01u8, 0x23, 0x01, 0x23, 0x45]; + let n = NibbleSlice::new(D); + let m = NibbleSlice::new(other); + + assert!(n != m); + assert!(n > m); + assert!(m < n); + + assert!(n == m.mid(4)); + assert!(n >= m.mid(4)); + assert!(n <= m.mid(4)); + } +} \ No newline at end of file From c79a2fc4ea9400e450323cfbe929b829bdc8ea61 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 29 Nov 2015 21:35:21 +0100 Subject: [PATCH 111/379] Examples and docs. --- src/nibbleslice.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 35068f0a2..e49244325 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -1,8 +1,29 @@ +//! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. use std::cmp::*; /// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. /// /// This is an immutable struct. No operations actually change it. +/// +/// # Example +/// ```rust +/// extern crate ethcore_util; +/// use ethcore_util::nibbleslice::*; +/// fn main() { +/// let d1 = &[0x01u8, 0x23, 0x45]; +/// let d2 = &[0x34u8, 0x50, 0x12]; +/// let d3 = &[0x00u8, 0x12]; +/// let n1 = NibbleSlice::new(d1); // 0,1,2,3,4,5 +/// let n2 = NibbleSlice::new(d2); // 3,4,5,0,1,2 +/// let n3 = NibbleSlice::new_offset(d3, 1); // 0,1,2 +/// assert!(n1 > n3); // 0,1,2,... > 0,1,2 +/// assert!(n1 < n2); // 0,... < 3,... +/// assert!(n2.mid(3) == n3); // 0,1,2 == 0,1,2 +/// assert!(n1.starts_with(&n3)); +/// assert_eq!(n1.common_prefix(&n3), 3); +/// assert_eq!(n2.mid(3).common_prefix(&n1), 3); +/// } +/// ``` #[derive(Debug, Copy, Clone, Eq, Ord)] pub struct NibbleSlice<'a> { data: &'a [u8], From 0c9ef317b23b95df55f7085a5b7555626759bd6c Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 00:28:03 +0100 Subject: [PATCH 112/379] fixed encoding of bytes in rlp, fixed encoding of long lists, triehash + basic tests... more tests todo --- src/rlp.rs | 39 +++++++++++--- src/triehash.rs | 136 ++++++++++++++++++++++++++++++++++++++++-------- src/vector.rs | 14 +++++ 3 files changed, 161 insertions(+), 28 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index feae875e3..1f791b3b7 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -456,9 +456,8 @@ impl RlpStream { } /// Apends value to the end of stream, chainable. - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream - where E: Encodable - { + pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug { + //println!("append: {:?}", object); // encode given value and add it at the end of the stream object.encode(&mut self.encoder); @@ -471,6 +470,7 @@ impl RlpStream { /// Declare appending the list of given size, chainable. pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { + //println!("append_list: {}", len); // push new list let position = self.encoder.bytes.len(); match len { @@ -488,6 +488,7 @@ impl RlpStream { /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { + //println!("append_raw: {:?} len: {}, count: {}", bytes, bytes.len(), item_count); // push raw items self.encoder.bytes.extend(bytes); @@ -623,7 +624,7 @@ impl BasicEncoder { match len { 0...55 => res.push(0xc0u8 + len as u8), _ => { - res.push(0x7fu8 + len.to_bytes_len() as u8); + res.push(0xf7u8 + len.to_bytes_len() as u8); res.extend(len.to_bytes()); } }; @@ -658,8 +659,7 @@ impl Encoder for BasicEncoder { } } - fn emit_list(&mut self, f: F) -> () - where F: FnOnce(&mut Self) -> () + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () { // get len before inserting a list let before_len = self.bytes.len(); @@ -891,6 +891,33 @@ mod tests { assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); } + #[test] + fn rlp_stream_list2() { + let mut stream = RlpStream::new(); + stream.append_list(17); + for _ in 0..17 { + stream.append(&""); + } + let out = stream.out().unwrap(); + assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); + } + + #[test] + fn rlp_stream_list3() { + let mut stream = RlpStream::new(); + stream.append_list(17); + + let mut res = vec![0xf8, 0x44]; + for _ in 0..17 { + stream.append(&"aaa"); + res.extend(vec![0x83, b'a', b'a', b'a']); + } + let out = stream.out().unwrap(); + assert_eq!(out, res); + } + struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { diff --git a/src/triehash.rs b/src/triehash.rs index ad553cf70..ff84695e4 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -139,6 +139,20 @@ impl NibblePair { } } +fn shared_nibble_prefix_len(vec: &[NibblePair]) -> usize { + if vec.len() == 0 { + return 0; + } + + vec.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first and each successive + .fold(vec[0].nibble.len(), | acc, pair | { + cmp::min(vec[0].nibble.shared_prefix_len(&pair.nibble), acc) + }) +} + pub fn ordered_trie_root(data: Vec>) -> H256 { let vec: Vec = data // first put elements into btree to sort them by nibbles @@ -167,7 +181,6 @@ pub fn hash256(vec: &[NibblePair]) -> H256 { } }; - println!("out: {:?}", out); out.sha3() } @@ -182,13 +195,7 @@ fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { stream.append(&vec[0].data); }, _ => { - let shared_prefix = vec.iter() - // skip first element - .skip(1) - // get minimum number of shared nibbles between first and each successive - .fold(usize::max_value(), | acc, pair | { - cmp::min(vec[0].nibble.shared_prefix_len(&pair.nibble), acc) - }); + let shared_prefix = shared_nibble_prefix_len(vec); match shared_prefix > pre_len { true => { @@ -199,12 +206,12 @@ fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { false => { stream.append_list(17); - // every nibble is longer then previous one - let iter = vec.iter() - // move to first element with len different then pre_len - .take_while(| pair | { pair.nibble.len() == pre_len }); - - let mut begin = iter.count(); + // if first nibble len is equal to pre_len + // move forward + let mut begin = match pre_len == vec[0].nibble.len() { + true => 1, + false => 0 + }; for i in 0..16 { // cout how many successive elements have same next nibble @@ -214,14 +221,14 @@ fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { match len { 0 => { stream.append(&""); }, - _ => hash256aux(&vec[begin..begin + len], pre_len + 1, stream) + _ => hash256aux(&vec[begin..(begin + len)], pre_len + 1, stream) } begin += len; } match pre_len == vec[0].nibble.len() { - true => stream.append(&vec[0].data), - false => stream.append(&"") + true => { stream.append(&vec[0].data); } , + false => { stream.append(&""); } }; } } @@ -239,6 +246,39 @@ fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { }; } + +#[test] +fn test_shared_nibble_len() { + let len = shared_nibble_prefix_len(&vec![ + NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), + NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), + ]); + + assert_eq!(len , 7); +} + +#[test] +fn test_shared_nibble_len2() { + let len = shared_nibble_prefix_len(&vec![ + NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), + NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), + NibblePair::new(vec![4, 1, 2, 3, 4, 5, 6], vec![]) + ]); + + assert_eq!(len , 0); +} + +#[test] +fn test_shared_nibble_len3() { + let len = shared_nibble_prefix_len(&vec![ + NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), + NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), + NibblePair::new(vec![0, 1, 2, 4, 4, 5, 6], vec![]) + ]); + + assert_eq!(len , 3); +} + #[cfg(test)] mod tests { use std::str::FromStr; @@ -246,6 +286,8 @@ mod tests { use hash::*; use triehash::*; + + #[test] fn empty_trie_root() { assert_eq!(hash256(&vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); @@ -262,9 +304,60 @@ mod tests { assert_eq!(hash256(&v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); } + #[test] + fn foo_trie_item() { + + let v = vec![ + NibblePair::new_raw(From::from("foo"), + From::from("bar")), + NibblePair::new_raw(From::from("food"), + From::from("bass")) + ]; + + assert_eq!(hash256(&v), H256::from_str("17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3").unwrap()); + } + + #[test] + fn dogs_trie_item() { + + let v = vec![ + NibblePair::new_raw(From::from("doe"), + From::from("reindeer")), + + NibblePair::new_raw(From::from("dog"), + From::from("puppy")), + + NibblePair::new_raw(From::from("dogglesworth"), + From::from("cat")), + ]; + + assert_eq!(hash256(&v), H256::from_str("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3").unwrap()); + } + + #[test] + fn puppy_trie_items() { + + let v = vec![ + NibblePair::new_raw(From::from("do"), + From::from("verb")), + + NibblePair::new_raw(From::from("dog"), + From::from("puppy")), + + NibblePair::new_raw(From::from("doge"), + From::from("coin")), + + NibblePair::new_raw(From::from("horse"), + From::from("stallion")), + + ]; + + assert_eq!(hash256(&v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); + } + #[test] fn test_trie_root() { - let _v = vec![ + let v = vec![ NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), @@ -290,10 +383,9 @@ mod tests { "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) ]; - let _root = H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap(); + let root = H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap(); - //let res = hash256(&v); - //println!("{:?}", res); - //assert_eq!(res, root); + let res = hash256(&v); + assert_eq!(res, root); } } diff --git a/src/vector.rs b/src/vector.rs index 41c932c92..552e285cf 100644 --- a/src/vector.rs +++ b/src/vector.rs @@ -64,8 +64,22 @@ mod test { #[test] fn test_shared_prefix() { + let a = vec![1,2,3,4,5,6]; + let b = vec![4,2,3,4,5,6]; + assert_eq!(a.shared_prefix_len(&b), 0); + } + + #[test] + fn test_shared_prefix2() { let a = vec![1,2,3,3,5]; let b = vec![1,2,3]; assert_eq!(a.shared_prefix_len(&b), 3); } + + #[test] + fn test_shared_prefix3() { + let a = vec![1,2,3,4,5,6]; + let b = vec![1,2,3,4,5,6]; + assert_eq!(a.shared_prefix_len(&b), 6); + } } From 710d3594a8e823a43dc07ca83767d9e52a0bbcc6 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 01:19:04 +0100 Subject: [PATCH 113/379] triehash clenaup in progress --- src/triehash.rs | 106 +++++++++++++++++++++++++++--------------------- 1 file changed, 59 insertions(+), 47 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index ff84695e4..abc13a39e 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -139,7 +139,7 @@ impl NibblePair { } } -fn shared_nibble_prefix_len(vec: &[NibblePair]) -> usize { +fn nibble_shared_prefix_len(vec: &[NibblePair]) -> usize { if vec.len() == 0 { return 0; } @@ -185,54 +185,66 @@ pub fn hash256(vec: &[NibblePair]) -> H256 { } fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { - match vec.len() { - 0 => { - stream.append(&""); - }, - 1 => { - stream.append_list(2); - stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..], true)); - stream.append(&vec[0].data); - }, - _ => { - let shared_prefix = shared_nibble_prefix_len(vec); + let vlen = vec.len(); - match shared_prefix > pre_len { - true => { - stream.append_list(2); - stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..shared_prefix], false)); - hash256aux(vec, shared_prefix, stream); - }, - false => { - stream.append_list(17); + // in case of empty slice, just append null + if vlen == 0 { + stream.append(&""); + return; + } - // if first nibble len is equal to pre_len - // move forward - let mut begin = match pre_len == vec[0].nibble.len() { - true => 1, - false => 0 - }; + // if the slice contains just one item, append the suffix of the key + // and then append value + if vlen == 1 { + stream.append_list(2); + stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..], true)); + stream.append(&vec[0].data); + return; + } - for i in 0..16 { - // cout how many successive elements have same next nibble - let len = vec[begin..].iter() - .map(| pair | pair.nibble[pre_len] ) - .take_while(|&q| q == i).count(); - - match len { - 0 => { stream.append(&""); }, - _ => hash256aux(&vec[begin..(begin + len)], pre_len + 1, stream) - } - begin += len; - } + // get length of the longest shared prefix in slice keys + let shared_prefix = nibble_shared_prefix_len(vec); - match pre_len == vec[0].nibble.len() { - true => { stream.append(&vec[0].data); } , - false => { stream.append(&""); } - }; - } - } + // if shared prefix is higher than current prefix append its + // new part of the key to the stream + // then recursively append suffixes of all items who had this key + if shared_prefix > pre_len { + stream.append_list(2); + stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..shared_prefix], false)); + hash256aux(vec, shared_prefix, stream); + return; + } + + // an item for every possible nibble/suffix + // + 1 for data + stream.append_list(17); + + // if first key len is equal to prefix_len, move to next element + let mut begin = match pre_len == vec[0].nibble.len() { + true => 1, + false => 0 + }; + + // iterate over all possible nibbles + for i in 0..16 { + // cout how many successive elements have same next nibble + let len = vec[begin..].iter() + .map(| pair | pair.nibble[pre_len] ) + .take_while(|&q| q == i).count(); + + // if at least 1 successive element has the same nibble + // append their suffixes + match len { + 0 => { stream.append(&""); }, + _ => hash256aux(&vec[begin..(begin + len)], pre_len + 1, stream) } + begin += len; + } + + // if fist key len is equal prefix, append it's value + match pre_len == vec[0].nibble.len() { + true => { stream.append(&vec[0].data); }, + false => { stream.append(&""); } }; } @@ -249,7 +261,7 @@ fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { #[test] fn test_shared_nibble_len() { - let len = shared_nibble_prefix_len(&vec![ + let len = nibble_shared_prefix_len(&vec![ NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), ]); @@ -259,7 +271,7 @@ fn test_shared_nibble_len() { #[test] fn test_shared_nibble_len2() { - let len = shared_nibble_prefix_len(&vec![ + let len = nibble_shared_prefix_len(&vec![ NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), NibblePair::new(vec![4, 1, 2, 3, 4, 5, 6], vec![]) @@ -270,7 +282,7 @@ fn test_shared_nibble_len2() { #[test] fn test_shared_nibble_len3() { - let len = shared_nibble_prefix_len(&vec![ + let len = nibble_shared_prefix_len(&vec![ NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), NibblePair::new(vec![0, 1, 2, 4, 4, 5, 6], vec![]) From a67abd8e993350f113651cb2926ee86ef360d5ce Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 01:21:37 +0100 Subject: [PATCH 114/379] upgraded travis to beta --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 81a39492f..eedbcaaad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: rust rust: - - 1.4.0 + - beta os: #- linux From 448577d99cf2ab9b8024f82e9e953cbedef1357a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 01:30:35 +0100 Subject: [PATCH 115/379] is_value -> is_data --- src/rlp.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 1f791b3b7..8e3553252 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -190,8 +190,8 @@ impl<'a> Rlp<'a> { } /// returns true if rlp is a value - pub fn is_value(&self) -> bool { - self.rlp.is_value() + pub fn is_data(&self) -> bool { + self.rlp.is_data() } /// returns rlp iterator @@ -242,7 +242,7 @@ impl<'a> UntrustedRlp<'a> { } /// returns true if rlp is a value - pub fn is_value(&self) -> bool { + pub fn is_data(&self) -> bool { self.bytes.len() > 0 && self.bytes[0] <= 0xbf } @@ -357,7 +357,7 @@ pub trait Decodable: Sized { impl Decodable for T where T: FromBytes { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_value() { + match rlp.is_data() { true => BasicDecoder::read_value(rlp.bytes, | bytes | { Ok(try!(T::from_bytes(bytes))) }), @@ -377,7 +377,7 @@ impl Decodable for Vec where T: Decodable { impl Decodable for Vec { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_value() { + match rlp.is_data() { true => BasicDecoder::read_value(rlp.bytes, | bytes | { let mut res = vec![]; res.extend(bytes); @@ -694,17 +694,17 @@ mod tests { assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); - assert!(cat.is_value()); + assert!(cat.is_data()); assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); - assert!(dog.is_value()); + assert!(dog.is_data()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_value()); + assert!(cat_again.is_data()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); } @@ -733,18 +733,18 @@ mod tests { let mut iter = rlp.iter(); let cat = iter.next().unwrap(); - assert!(cat.is_value()); + assert!(cat.is_data()); assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); let dog = iter.next().unwrap(); - assert!(dog.is_value()); + assert!(dog.is_data()); assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); let none = iter.next(); assert!(none.is_none()); let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_value()); + assert!(cat_again.is_data()); assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); } } From 122144c00a69740fac44055702785e3c5c2fdd42 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 02:55:03 +0100 Subject: [PATCH 116/379] docs for rlp --- src/rlp.rs | 358 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 248 insertions(+), 110 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 8e3553252..a24c63b46 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1,85 +1,34 @@ -//! Rlp serialization module -//! -//! Types implementing `Endocable` and `Decodable` traits -//! can be easily coverted to and from rlp. -//! Trusted rlp should be decoded with `Rlp`, untrusted with `UntrustedRlp`. -//! -//! # Examples: -//! -//! ```rust -//! extern crate ethcore_util; -//! use ethcore_util::rlp::{Rlp, UntrustedRlp, RlpStream, Decodable}; -//! -//! fn encode_value() { -//! // 1029 -//! let mut stream = RlpStream::new(); -//! stream.append(&1029u32); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0x82, 0x04, 0x05]); -//! } -//! -//! fn encode_list() { -//! // [ "cat", "dog" ] -//! let mut stream = RlpStream::new_list(2); -//! stream.append(&"cat").append(&"dog"); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); -//! } -//! -//! fn encode_list2() { -//! // [ [], [[]], [ [], [[]] ] ] -//! let mut stream = RlpStream::new_list(3); -//! stream.append_list(0); -//! stream.append_list(1).append_list(0); -//! stream.append_list(2).append_list(0).append_list(1).append_list(0); -//! let out = stream.out().unwrap(); -//! assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); -//! } -//! -//! fn decode_value() { -//! // 0x102456789abcdef -//! let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; -//! let rlp = Rlp::new(&data); -//! let _ = u64::decode(&rlp); -//! } +//! Rlp serialization module //! -//! fn decode_untrusted_string() { -//! // "cat" -//! let data = vec![0x83, b'c', b'a', b't']; -//! let rlp = UntrustedRlp::new(&data); -//! let _ = String::decode_untrusted(&rlp).unwrap(); -//! } -//! -//! fn decode_list() { -//! // ["cat", "dog"] -//! let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -//! let rlp = Rlp::new(&data); -//! let _ : Vec = Decodable::decode(&rlp); -//! } -//! -//! fn decode_list2() { -//! // [ [], [[]], [ [], [[]] ] ] -//! let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; -//! let rlp = Rlp::new(&data); -//! let _v0: Vec = Decodable::decode(&rlp.at(0)); -//! let _v1: Vec> = Decodable::decode(&rlp.at(1)); -//! let nested_rlp = rlp.at(2); -//! let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); -//! let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); -//! } +//! Allows encoding, decoding, and view onto rlp-slice //! -//! fn main() { -//! encode_value(); -//! encode_list(); -//! encode_list2(); +//!# When should you use what? //! -//! decode_value(); -//! decode_untrusted_string(); -//! decode_list(); -//! decode_list2(); -//! } -//! ``` +//!### Use `encode` function when: +//! * You want to encode something inline. +//! * You do not work on big set of data. +//! * You want to encode whole data structure at once. //! +//!### Use `decode` function when: +//! * You want to decode something inline. +//! * You do not work on big set of data. +//! * You want to decode whole rlp at once. +//! +//!### Use `RlpStream` when: +//! * You want to encode something in portions. +//! * You encode a big set of data. +//! +//!### Use `Rlp` when: +//! * You are working on trusted data (not corrupted). +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. +//! +//!### Use `UntrustedRlp` when: +//! * You are working on untrusted data (~corrupted). +//! * You need to handle data corruption errors. +//! * You are working on input data. +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. use std::fmt; use std::cell::Cell; @@ -88,7 +37,12 @@ use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; use vector::InsertSlice; -/// rlp container +/// Data-oriented view onto rlp-slice. +/// +/// This is immutable structere. No operations change it. +/// +/// Should be used in places where, error handling is required, +/// eg. on input #[derive(Debug)] pub struct UntrustedRlp<'a> { bytes: &'a [u8], @@ -152,10 +106,10 @@ impl From for DecoderError { } } -/// Unsafe wrapper for rlp decode_untrustedr. +/// Data-oriented view onto trusted rlp-slice. /// -/// It assumes that you know what you are doing. Doesn't bother -/// you with error handling. +/// Unlikely to `UntrustedRlp` doesn't bother you with error +/// handling. It assumes that you know what you are doing. pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } @@ -173,30 +127,79 @@ impl<'a> From> for UntrustedRlp<'a> { } impl<'a> Rlp<'a> { - /// returns new instance of `Rlp` + /// Create a new instance of `Rlp` pub fn new(bytes: &'a [u8]) -> Rlp<'a> { Rlp { rlp: UntrustedRlp::new(bytes) } } + /// Get view onto rlp-slice at index. + /// + /// Caches offset to given index, so access to successive + /// slices is faster. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let dog = String::decode(&rlp.at(1)); + /// assert_eq!(dog, "dog".to_string()); + /// } + /// ``` pub fn at(&self, index: usize) -> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } - /// returns true if rlp is a list + /// List value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_list()); + /// } + /// ``` pub fn is_list(&self) -> bool { self.rlp.is_list() } - /// returns true if rlp is a value + /// String value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.at(1).is_data()); + /// } + /// ``` pub fn is_data(&self) -> bool { self.rlp.is_data() } - /// returns rlp iterator - pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { - self.rlp.into_iter() + /// Get iterator over rlp-slices + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); + /// } + /// ``` + pub fn iter(&'a self) -> RlpIterator<'a> { + self.into_iter() } } @@ -209,9 +212,22 @@ impl<'a> UntrustedRlp<'a> { } } - /// get container subset at given index - /// - /// paren container caches searched position + /// Get view onto rlp-slice at index + /// + /// Caches offset to given index, so access to successive + /// slices is faster. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// let dog = String::decode_untrusted(&rlp.at(1).unwrap()).unwrap(); + /// assert_eq!(dog, "dog".to_string()); + /// } + /// ``` pub fn at(&self, index: usize) -> Result, DecoderError> { if !self.is_list() { return Err(DecoderError::UntrustedRlpExpectedToBeList); @@ -236,17 +252,53 @@ impl<'a> UntrustedRlp<'a> { Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len])) } - /// returns true if rlp is a list + /// List value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.is_list()); + /// } + /// ``` pub fn is_list(&self) -> bool { self.bytes.len() > 0 && self.bytes[0] >= 0xc0 } - /// returns true if rlp is a value + /// String value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.at(1).unwrap().is_data()); + /// } + /// ``` pub fn is_data(&self) -> bool { self.bytes.len() > 0 && self.bytes[0] <= 0xbf } - /// returns rlp iterator + /// Get iterator over rlp-slices + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// let strings: Vec = rlp.iter() + /// .map(| i | String::decode_untrusted(&i)) + /// .map(| s | s.unwrap()) + /// .collect(); + /// } + /// ``` pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { self.into_iter() } @@ -307,7 +359,7 @@ impl<'a> UntrustedRlp<'a> { } } -/// non-consuming rlp iterator +/// Iterator over rlp-slice list elements. pub struct UntrustedRlpIterator<'a> { rlp: &'a UntrustedRlp<'a>, index: usize, @@ -336,18 +388,53 @@ impl<'a> Iterator for UntrustedRlpIterator<'a> { } } -/// shortcut function to decoded Trusted Rlp `&[u8]` into an object + +/// Iterator over trusted rlp-slice list elements. +pub struct RlpIterator<'a> { + rlp: &'a Rlp<'a>, + index: usize +} + +impl<'a> IntoIterator for &'a Rlp<'a> { + type Item = Rlp<'a>; + type IntoIter = RlpIterator<'a>; + + fn into_iter(self) -> Self::IntoIter { + RlpIterator { + rlp: self, + index: 0, + } + } +} + +impl<'a> Iterator for RlpIterator<'a> { + type Item = Rlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.rlp.at(index).ok().map(| iter | { From::from(iter) }); + self.index += 1; + result + } +} + +/// Shortcut function to decode trusted rlp +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +/// let animals: Vec = decode(&data); +/// assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); +/// } +/// ``` pub fn decode(bytes: &[u8]) -> T where T: Decodable { let rlp = Rlp::new(bytes); T::decode(&rlp) } -/// shortcut function to decode UntrustedRlp `&[u8]` into an object -pub fn decode_untrusted(bytes: &[u8]) -> Result where T: Decodable { - let rlp = UntrustedRlp::new(bytes); - T::decode_untrusted(&rlp) -} - pub trait Decodable: Sized { fn decode_untrusted(rlp: &UntrustedRlp) -> Result; fn decode(rlp: &Rlp) -> Self { @@ -432,15 +519,14 @@ impl ListInfo { } } -/// container that should be used to encode rlp +/// Appendable rlp encoder. pub struct RlpStream { unfinished_lists: LinkedList, encoder: BasicEncoder, } impl RlpStream { - /// create new container for values appended one after another, - /// but not being part of the same list + /// Initializes instance of empty `RlpStream`. pub fn new() -> RlpStream { RlpStream { unfinished_lists: LinkedList::new(), @@ -448,7 +534,7 @@ impl RlpStream { } } - /// create new container for list of size `max_len` + /// Initializes the `RLPStream` as a list. pub fn new_list(len: usize) -> RlpStream { let mut stream = RlpStream::new(); stream.append_list(len); @@ -456,6 +542,18 @@ impl RlpStream { } /// Apends value to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append(&"dog"); + /// let out = stream.out().unwrap(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + /// ``` pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug { //println!("append: {:?}", object); // encode given value and add it at the end of the stream @@ -469,6 +567,19 @@ impl RlpStream { } /// Declare appending the list of given size, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_list(2).append(&"cat").append(&"dog"); + /// stream.append(&""); + /// let out = stream.out().unwrap(); + /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); + /// } + /// ``` pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { //println!("append_list: {}", len); // push new list @@ -499,12 +610,28 @@ impl RlpStream { self } - /// return true if stream is ready + /// Returns true if stream doesnt expect any more items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat"); + /// assert_eq!(stream.is_finished(), false); + /// stream.append(&"dog"); + /// assert_eq!(stream.is_finished(), true); + /// let out = stream.out().unwrap(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } pub fn is_finished(&self) -> bool { self.unfinished_lists.back().is_none() } - /// streams out encoded bytes + /// Streams out encoded bytes. + /// + /// Returns an error if stream is not finished. pub fn out(self) -> Result, EncoderError> { match self.is_finished() { true => Ok(self.encoder.out()), @@ -534,7 +661,18 @@ impl RlpStream { } } -/// shortcut function to encode a `T: Encodable` into a UntrustedRlp `Vec` +/// Shortcut function to encode structure into rlp. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let animals = vec!["cat", "dog"]; +/// let out = encode(&animals); +/// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); +/// } +/// ``` pub fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); @@ -922,7 +1060,7 @@ mod tests { fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { for t in &tests { - let res: T = rlp::decode_untrusted(&t.1).unwrap(); + let res: T = rlp::decode(&t.1); assert_eq!(res, t.0); } } From c91bc6b419722199392381fff690919e1dd62e6d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 30 Nov 2015 02:57:02 +0100 Subject: [PATCH 117/379] Bug fix in nibble slice. Trie. --- src/hashdb.rs | 4 +++ src/memorydb.rs | 24 +++++++++++-- src/nibbleslice.rs | 22 ++++++++++++ src/overlaydb.rs | 1 + src/trie.rs | 86 +++++++++++++++++++++++++++++++++++++--------- 5 files changed, 119 insertions(+), 18 deletions(-) diff --git a/src/hashdb.rs b/src/hashdb.rs index 561c7843d..dd0973bb3 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -1,4 +1,5 @@ use hash::*; +use bytes::*; pub trait HashDB { /// Look up a given hash into the bytes that hash to it, returning None if the @@ -56,6 +57,9 @@ pub trait HashDB { /// ``` fn insert(&mut self, value: &[u8]) -> H256; + /// Like `insert()` , except you provide the key and the data is all moved. + fn emplace(&mut self, key: H256, value: Bytes); + /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may /// happen without the data being eventually being inserted into the DB. /// diff --git a/src/memorydb.rs b/src/memorydb.rs index 4ec02bd5e..14fae12d0 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -37,6 +37,12 @@ use std::collections::HashMap; /// m.kill(&k); /// assert!(!m.exists(&k)); /// +/// m.kill(&k); +/// assert!(!m.exists(&k)); +/// +/// m.insert(d); +/// assert!(!m.exists(&k)); + /// m.insert(d); /// assert!(m.exists(&k)); /// assert_eq!(m.lookup(&k).unwrap(), d); @@ -130,9 +136,9 @@ impl HashDB for MemoryDB { fn insert(&mut self, value: &[u8]) -> H256 { let key = value.sha3(); if match self.data.get_mut(&key) { - Some(&mut (ref mut old_value, ref mut rc @ 0)) => { + Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { *old_value = From::from(value.bytes()); - *rc = 1; + *rc += 1; false }, Some(&mut (_, ref mut x)) => { *x += 1; false } , @@ -143,6 +149,20 @@ impl HashDB for MemoryDB { key } + fn emplace(&mut self, key: H256, value: Bytes) { + match self.data.get_mut(&key) { + Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { + *old_value = value; + *rc += 1; + return; + }, + Some(&mut (_, ref mut x)) => { *x += 1; return; } , + None => {}, + } + // ... None falls through into... + self.data.insert(key, (value, 1)); + } + fn kill(&mut self, key: &H256) { if match self.data.get_mut(key) { Some(&mut (_, ref mut x)) => { *x -= 1; false } diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index e49244325..5866ed8f0 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -1,5 +1,6 @@ //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. use std::cmp::*; +use bytes::*; /// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. /// @@ -69,6 +70,18 @@ impl<'a> NibbleSlice<'a> { } i } + + pub fn encoded(&self, is_leaf: bool) -> Bytes { + let l = self.len(); + let mut r = Bytes::with_capacity(l / 2 + 1); + let mut i = l % 2; + r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0}); + while i < l { + r.push(self.at(i) * 16 + self.at(i + 1)); + i += 2; + } + r + } } impl<'a> PartialEq for NibbleSlice<'a> { @@ -126,6 +139,15 @@ mod tests { } } + #[test] + fn encoded() { + let n = NibbleSlice::new(D); + assert_eq!(n.encoded(false), &[0x00, 0x01, 0x23, 0x45]); + assert_eq!(n.encoded(true), &[0x20, 0x01, 0x23, 0x45]); + assert_eq!(n.mid(1).encoded(false), &[0x11, 0x23, 0x45]); + assert_eq!(n.mid(1).encoded(true), &[0x31, 0x23, 0x45]); + } + #[test] fn shared() { let n = NibbleSlice::new(D); diff --git a/src/overlaydb.rs b/src/overlaydb.rs index b6f2b3eb2..3c35d7255 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -186,6 +186,7 @@ impl HashDB for OverlayDB { } } fn insert(&mut self, value: &[u8]) -> H256 { self.overlay.insert(value) } + fn emplace(&mut self, key: H256, value: Bytes) { self.overlay.emplace(key, value); } fn kill(&mut self, key: &H256) { self.overlay.kill(key); } } diff --git a/src/trie.rs b/src/trie.rs index 889c9b5dc..80e753f84 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,8 +1,9 @@ use memorydb::*; use hashdb::*; use hash::*; -//use rlp::*; -//use bytes::*; +use nibbleslice::*; +use bytes::*; +use rlp::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); @@ -17,10 +18,10 @@ pub trait Trie { fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } // TODO: consider returning &[u8]... - fn contains(key: &[u8]) -> bool; - fn at(key: &[u8]) -> Option<&[u8]>; - fn insert(key: &[u8], value: &[u8]); - fn remove(key: &[u8]); + fn contains(&self, key: &[u8]) -> bool; + fn at(&self, key: &[u8]) -> Option<&[u8]>; + fn insert(&mut self, key: &[u8], value: &[u8]); + fn remove(&mut self, key: &[u8]); } pub struct TrieDB { @@ -28,6 +29,11 @@ pub struct TrieDB { root: H256, } +struct Diff { + new: Vec<(H256, Bytes)>, + old: Vec, +} + impl TrieDB { pub fn new(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } } @@ -35,36 +41,85 @@ impl TrieDB { pub fn new_memory() -> Self { TrieDB{ db: Box::new(MemoryDB::new()), root: H256::new() } } - pub fn init(&mut self) { self.insert_root(&NULL_RLP); } + pub fn init(&mut self) { self.set_root_rlp(&NULL_RLP); } pub fn db(&self) -> &HashDB { self.db.as_ref() } - fn insert_root(&mut self, root_data: &[u8]) { self.root = self.db.insert(root_data); } + fn set_root_rlp(&mut self, root_data: &[u8]) { + self.root = self.db.insert(root_data); + } + fn insert(&mut self, key: &NibbleSlice, value: &[u8]) { + // determine what the new root is, insert new nodes and remove old as necessary. + let mut todo: (Bytes, Diff); + { + let root_rlp = self.db.lookup(&self.root).unwrap(); + todo = self.merge(root_rlp, key, value); + } + self.apply(todo.1); + self.set_root_rlp(&todo.0); + } + + fn apply(&mut self, diff: Diff) { + for d in diff.old.iter() { + self.db.kill(&d); + } + for d in diff.new.into_iter() { + self.db.emplace(d.0, d.1); + } + } + + /// Determine the RLP of the node, assuming we're inserting `partial_key` into the + /// node at `old`. This will *not* delete the old mode; it will just return the new RLP + /// that includes the new node. + /// + /// The database will be updated so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + fn merge(&self, old: &[u8], partial_key: &NibbleSlice, value: &[u8]) -> (Bytes, Diff) { + let o = Rlp::new(old); + match (o.type()) { + List(17) => { + // already have a branch. route and merge. + }, + List(2) => { + // already have an extension. either fast_forward, cleve or transmute_to_branch. + }, + Data(0) => compose_extension(partial_key), + _ -> panic!("Invalid RLP for node."), + } + } + + fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { + let mut s = RlpStream::new_list(2); + s.append(&partial_key.encoded(is_leaf)); + s.append(&value.to_vec()); // WTF?!?! + //s.append(value); // <-- should be. + s.out().unwrap() + } } impl Trie for TrieDB { fn root(&self) -> &H256 { &self.root } - fn contains(_key: &[u8]) -> bool { + fn contains(&self, _key: &[u8]) -> bool { unimplemented!(); } - fn at(_key: &[u8]) -> Option<&[u8]> { + fn at(&self, _key: &[u8]) -> Option<&[u8]> { unimplemented!(); } - fn insert(_key: &[u8], _value: &[u8]) { - unimplemented!(); + fn insert(&mut self, key: &[u8], value: &[u8]) { + (self as &mut TrieDB).insert(&NibbleSlice::new(key), value); } - fn remove(_key: &[u8]) { + fn remove(&mut self, _key: &[u8]) { unimplemented!(); } } #[test] -fn it_works() { +fn playpen() { use overlaydb::*; (&[1, 2, 3]).starts_with(&[1, 2]); @@ -73,6 +128,5 @@ fn it_works() { t.init(); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); - // TODO: make work: - //assert_eq!(t.root(), SHA3_NULL_RLP); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); } \ No newline at end of file From 78965b3c5aaff784bdb6ee9b1cd82884b9f2c7ea Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:16:50 +0100 Subject: [PATCH 118/379] RlpStream append_null --- src/rlp.rs | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index a24c63b46..f76f1f5b9 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -2,7 +2,7 @@ //! //! Allows encoding, decoding, and view onto rlp-slice //! -//!# When should you use what? +//!# What should you use when? //! //!### Use `encode` function when: //! * You want to encode something inline. @@ -555,7 +555,6 @@ impl RlpStream { /// } /// ``` pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug { - //println!("append: {:?}", object); // encode given value and add it at the end of the stream object.encode(&mut self.encoder); @@ -581,7 +580,6 @@ impl RlpStream { /// } /// ``` pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { - //println!("append_list: {}", len); // push new list let position = self.encoder.bytes.len(); match len { @@ -597,9 +595,32 @@ impl RlpStream { self } + /// Apends null to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_null().append_null(); + /// let out = stream.out().unwrap(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); + /// } + /// ``` + pub fn append_null<'a>(&'a mut self) -> &'a mut RlpStream { + // self push raw item + self.encoder.bytes.push(0x80); + + // try to finish and prepend the length + self.try_to_finish(1); + + // return chainable self + self + } + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { - //println!("append_raw: {:?} len: {}, count: {}", bytes, bytes.len(), item_count); // push raw items self.encoder.bytes.extend(bytes); From 2d7ae3a18a22064f6c95df469ffc37f823fe3dda Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:24:03 +0100 Subject: [PATCH 119/379] RlpStream out returns value instead of Result --- src/overlaydb.rs | 2 +- src/rlp.rs | 43 +++++++++++++------------------------------ src/triehash.rs | 4 ++-- 3 files changed, 16 insertions(+), 33 deletions(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index b6f2b3eb2..24e4ea8da 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -130,7 +130,7 @@ impl OverlayDB { let mut s = RlpStream::new_list(2); s.append(&payload.1); s.append(&payload.0); - self.backing.put(&key.bytes(), &s.out().unwrap()).expect("Low-level database error. Some issue with your hard disk?"); + self.backing.put(&key.bytes(), &s.out()).expect("Low-level database error. Some issue with your hard disk?"); } } diff --git a/src/rlp.rs b/src/rlp.rs index f76f1f5b9..737f7f416 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -550,7 +550,7 @@ impl RlpStream { /// fn main () { /// let mut stream = RlpStream::new_list(2); /// stream.append(&"cat").append(&"dog"); - /// let out = stream.out().unwrap(); + /// let out = stream.out(); /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` @@ -575,7 +575,7 @@ impl RlpStream { /// let mut stream = RlpStream::new_list(2); /// stream.append_list(2).append(&"cat").append(&"dog"); /// stream.append(&""); - /// let out = stream.out().unwrap(); + /// let out = stream.out(); /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); /// } /// ``` @@ -604,7 +604,7 @@ impl RlpStream { /// fn main () { /// let mut stream = RlpStream::new_list(2); /// stream.append_null().append_null(); - /// let out = stream.out().unwrap(); + /// let out = stream.out(); /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); /// } /// ``` @@ -643,7 +643,7 @@ impl RlpStream { /// assert_eq!(stream.is_finished(), false); /// stream.append(&"dog"); /// assert_eq!(stream.is_finished(), true); - /// let out = stream.out().unwrap(); + /// let out = stream.out(); /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } pub fn is_finished(&self) -> bool { @@ -652,11 +652,11 @@ impl RlpStream { /// Streams out encoded bytes. /// - /// Returns an error if stream is not finished. - pub fn out(self) -> Result, EncoderError> { + /// panic! if stream is not finished + pub fn out(self) -> Vec { match self.is_finished() { - true => Ok(self.encoder.out()), - false => Err(EncoderError::StreamIsUnfinished), + true => self.encoder.out(), + false => panic!() } } @@ -701,23 +701,6 @@ pub fn encode(object: &E) -> Vec where E: Encodable encoder.out() } -#[derive(Debug)] -pub enum EncoderError { - StreamIsUnfinished, -} - -impl StdError for EncoderError { - fn description(&self) -> &str { - "encoder error" - } -} - -impl fmt::Display for EncoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - pub trait Encodable { fn encode(&self, encoder: &mut E) -> () where E: Encoder; } @@ -1035,7 +1018,7 @@ mod tests { fn rlp_stream() { let mut stream = RlpStream::new_list(2); stream.append(&"cat").append(&"dog"); - let out = stream.out().unwrap(); + let out = stream.out(); assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); } @@ -1046,7 +1029,7 @@ mod tests { stream.append_list(0); stream.append_list(1).append_list(0); stream.append_list(2).append_list(0).append_list(1).append_list(0); - let out = stream.out().unwrap(); + let out = stream.out(); assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); } @@ -1057,7 +1040,7 @@ mod tests { for _ in 0..17 { stream.append(&""); } - let out = stream.out().unwrap(); + let out = stream.out(); assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); @@ -1073,9 +1056,9 @@ mod tests { stream.append(&"aaa"); res.extend(vec![0x83, b'a', b'a', b'a']); } - let out = stream.out().unwrap(); + let out = stream.out(); assert_eq!(out, res); - } + }; struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; diff --git a/src/triehash.rs b/src/triehash.rs index abc13a39e..aaeef5f65 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -177,7 +177,7 @@ pub fn hash256(vec: &[NibblePair]) -> H256 { _ => { let mut stream = RlpStream::new(); hash256rlp(&vec, 0, &mut stream); - stream.out().unwrap() + stream.out() } }; @@ -251,7 +251,7 @@ fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { let mut s = RlpStream::new(); hash256rlp(vec, pre_len, &mut s); - let out = s.out().unwrap(); + let out = s.out(); match out.len() { 0...31 => stream.append_raw(&out, 1), _ => stream.append(&out.sha3()) From 94a5216513db4b71b64364d8db57360601b5cfa6 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:25:09 +0100 Subject: [PATCH 120/379] removed dangling ; --- src/rlp.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 737f7f416..27151295f 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -652,7 +652,7 @@ impl RlpStream { /// Streams out encoded bytes. /// - /// panic! if stream is not finished + /// Returns an error if stream is not finished. pub fn out(self) -> Vec { match self.is_finished() { true => self.encoder.out(), @@ -1058,7 +1058,7 @@ mod tests { } let out = stream.out(); assert_eq!(out, res); - }; + } struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; From 6ee5e1684b10853d5dcd3495e5e3aac5c569e45e Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:29:30 +0100 Subject: [PATCH 121/379] rename try_to_finish -> note_appended --- src/rlp.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 27151295f..846281a2b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -559,7 +559,7 @@ impl RlpStream { object.encode(&mut self.encoder); // if list is finished, prepend the length - self.try_to_finish(1); + self.note_appended(1); // return chainable self self @@ -586,7 +586,7 @@ impl RlpStream { 0 => { // we may finish, if the appended list len is equal 0 self.encoder.bytes.push(0xc0u8); - self.try_to_finish(1); + self.note_appended(1); } _ => self.unfinished_lists.push_back(ListInfo::new(position, len)), } @@ -613,7 +613,7 @@ impl RlpStream { self.encoder.bytes.push(0x80); // try to finish and prepend the length - self.try_to_finish(1); + self.note_appended(1); // return chainable self self @@ -625,7 +625,7 @@ impl RlpStream { self.encoder.bytes.extend(bytes); // try to finish and prepend the length - self.try_to_finish(item_count); + self.note_appended(item_count); // return chainable self self @@ -652,7 +652,7 @@ impl RlpStream { /// Streams out encoded bytes. /// - /// Returns an error if stream is not finished. + /// panic! if stream is not finished. pub fn out(self) -> Vec { match self.is_finished() { true => self.encoder.out(), @@ -660,8 +660,8 @@ impl RlpStream { } } - /// try to finish lists - fn try_to_finish(&mut self, inserted_items: usize) -> () { + /// Try to finish lists + fn note_appended(&mut self, inserted_items: usize) -> () { let should_finish = match self.unfinished_lists.back_mut() { None => false, Some(ref mut x) => { @@ -677,7 +677,7 @@ impl RlpStream { let x = self.unfinished_lists.pop_back().unwrap(); let len = self.encoder.bytes.len() - x.position; self.encoder.insert_list_len_at_pos(len, x.position); - self.try_to_finish(1); + self.note_appended(1); } } } From bd6289756bf0f1c348b8977c891b160bf10b63e2 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:33:08 +0100 Subject: [PATCH 122/379] fixed rlp decoder error names --- src/rlp.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 846281a2b..bf8cbd68e 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -83,10 +83,10 @@ impl ItemInfo { #[derive(Debug, PartialEq, Eq)] pub enum DecoderError { FromBytesError(FromBytesError), - UntrustedRlpIsTooShort, - UntrustedRlpExpectedToBeList, - UntrustedRlpExpectedToBeValue, - BadUntrustedRlp, + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, + BadRlp, } impl StdError for DecoderError { fn description(&self) -> &str { @@ -230,7 +230,7 @@ impl<'a> UntrustedRlp<'a> { /// ``` pub fn at(&self, index: usize) -> Result, DecoderError> { if !self.is_list() { - return Err(DecoderError::UntrustedRlpExpectedToBeList); + return Err(DecoderError::RlpExpectedToBeList); } // move to cached position if it's index is less or equal to @@ -325,7 +325,7 @@ impl<'a> UntrustedRlp<'a> { /// TODO: move this to decode_untrustedr? fn item_info(bytes: &[u8]) -> Result { let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::UntrustedRlpIsTooShort), + None => return Err(DecoderError::RlpIsTooShort), Some(0...0x7f) => ItemInfo::new(0, 1), Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), Some(l @ 0xb8...0xbf) => { @@ -341,12 +341,12 @@ impl<'a> UntrustedRlp<'a> { let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); ItemInfo::new(prefix_len, value_len) } - _ => return Err(DecoderError::BadUntrustedRlp), + _ => return Err(DecoderError::BadRlp), }; match item.prefix_len + item.value_len <= bytes.len() { true => Ok(item), - false => Err(DecoderError::UntrustedRlpIsTooShort), + false => Err(DecoderError::RlpIsTooShort), } } @@ -354,7 +354,7 @@ impl<'a> UntrustedRlp<'a> { fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { match bytes.len() >= len { true => Ok(&bytes[len..]), - false => Err(DecoderError::UntrustedRlpIsTooShort), + false => Err(DecoderError::RlpIsTooShort), } } } @@ -448,7 +448,7 @@ impl Decodable for T where T: FromBytes { true => BasicDecoder::read_value(rlp.bytes, | bytes | { Ok(try!(T::from_bytes(bytes))) }), - false => Err(DecoderError::UntrustedRlpExpectedToBeValue), + false => Err(DecoderError::RlpExpectedToBeData), } } } @@ -457,7 +457,7 @@ impl Decodable for Vec where T: Decodable { fn decode_untrusted(rlp: &UntrustedRlp) -> Result { match rlp.is_list() { true => rlp.iter().map(|rlp| T::decode_untrusted(&rlp)).collect(), - false => Err(DecoderError::UntrustedRlpExpectedToBeList), + false => Err(DecoderError::RlpExpectedToBeList), } } } @@ -470,7 +470,7 @@ impl Decodable for Vec { res.extend(bytes); Ok(res) }), - false => Err(DecoderError::UntrustedRlpExpectedToBeValue), + false => Err(DecoderError::RlpExpectedToBeData), } } } @@ -485,7 +485,7 @@ impl Decoder for BasicDecoder { fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { match bytes.first().map(|&x| x) { // rlp is too short - None => Err(DecoderError::UntrustedRlpIsTooShort), + None => Err(DecoderError::RlpIsTooShort), // single byt value Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), // 0-55 bytes @@ -497,7 +497,7 @@ impl Decoder for BasicDecoder { let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } - _ => Err(DecoderError::BadUntrustedRlp), + _ => Err(DecoderError::BadRlp), } } } @@ -860,10 +860,10 @@ mod tests { assert!(rlp.is_list()); let cat_err = rlp.at(0).unwrap_err(); - assert_eq!(cat_err, rlp::DecoderError::UntrustedRlpIsTooShort); + assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); let dog_err = rlp.at(1).unwrap_err(); - assert_eq!(dog_err, rlp::DecoderError::UntrustedRlpIsTooShort); + assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); } } From fac89f784b47f6c1dd9e353083f4c3ea294ab937 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:33:31 +0100 Subject: [PATCH 123/379] updated todo --- src/rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp.rs b/src/rlp.rs index bf8cbd68e..21f026116 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -322,7 +322,7 @@ impl<'a> UntrustedRlp<'a> { /// return first item info /// - /// TODO: move this to decode_untrustedr? + /// TODO: move this to decoder (?) fn item_info(bytes: &[u8]) -> Result { let item = match bytes.first().map(|&x| x) { None => return Err(DecoderError::RlpIsTooShort), From f42e8c06658829a90af2f1e1b7b17abea21d9c9d Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:42:54 +0100 Subject: [PATCH 124/379] RlpStream clear --- src/rlp.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index 21f026116..de63455fa 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -631,6 +631,28 @@ impl RlpStream { self } + /// Clear the output stream so far. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(3); + /// stream.append(&"cat"); + /// stream.clear(); + /// stream.append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); + /// } + pub fn clear(&mut self) { + // clear bytes + self.encoder.bytes.clear(); + + // clear lists + self.unfinished_lists.clear(); + } + /// Returns true if stream doesnt expect any more items. /// /// ```rust From 93f2bd93148ceed4270d27e61083cf6da055e996 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 11:54:09 +0100 Subject: [PATCH 125/379] Rlp data function --- src/rlp.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index de63455fa..deef44725 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -134,6 +134,24 @@ impl<'a> Rlp<'a> { } } + /// The bare data of the rlp. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let view = rlp.at(1); + /// let dog = view.data(); + /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); + /// } + /// ``` + pub fn data(&'a self) -> &'a [u8] { + self.rlp.data() + } + /// Get view onto rlp-slice at index. /// /// Caches offset to given index, so access to successive @@ -212,6 +230,24 @@ impl<'a> UntrustedRlp<'a> { } } + /// The bare data of the rlp. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// let view = rlp.at(1).unwrap(); + /// let dog = view.data(); + /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); + /// } + /// ``` + pub fn data(&'a self) -> &'a [u8] { + self.bytes + } + /// Get view onto rlp-slice at index /// /// Caches offset to given index, so access to successive From f4447295694325285bbecd8b581accffbf2f1755 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 12:06:08 +0100 Subject: [PATCH 126/379] Rlp is_null && is_empty methods --- src/rlp.rs | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index deef44725..fc0c2a25c 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -172,6 +172,38 @@ impl<'a> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } + /// No value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_null()); + /// } + /// ``` + pub fn is_null(&self) -> bool { + self.rlp.is_null() + } + + /// Contains a zero-length string or zero-length list. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc0]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + self.rlp.is_empty() + } + /// List value /// /// ```rust @@ -288,6 +320,38 @@ impl<'a> UntrustedRlp<'a> { Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len])) } + /// No value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![]; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.is_null()); + /// } + /// ``` + pub fn is_null(&self) -> bool { + self.bytes.len() == 0 + } + + /// Contains a zero-length string or zero-length list. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc0]; + /// let rlp = UntrustedRlp::new(&data); + /// assert!(rlp.is_empty()); + /// } + /// ``` + pub fn is_empty(&self) -> bool { + !self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80) + } + /// List value /// /// ```rust @@ -301,7 +365,7 @@ impl<'a> UntrustedRlp<'a> { /// } /// ``` pub fn is_list(&self) -> bool { - self.bytes.len() > 0 && self.bytes[0] >= 0xc0 + !self.is_null() && self.bytes[0] >= 0xc0 } /// String value @@ -317,7 +381,7 @@ impl<'a> UntrustedRlp<'a> { /// } /// ``` pub fn is_data(&self) -> bool { - self.bytes.len() > 0 && self.bytes[0] <= 0xbf + !self.is_null() && self.bytes[0] < 0xc0 } /// Get iterator over rlp-slices @@ -424,7 +488,6 @@ impl<'a> Iterator for UntrustedRlpIterator<'a> { } } - /// Iterator over trusted rlp-slice list elements. pub struct RlpIterator<'a> { rlp: &'a Rlp<'a>, From 80f1e527dc001f1488f1ba5b3e0ee9b01f3527ba Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 12:28:49 +0100 Subject: [PATCH 127/379] Rlp is_int --- src/rlp.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index fc0c2a25c..ba1b5498b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -236,6 +236,23 @@ impl<'a> Rlp<'a> { self.rlp.is_data() } + /// Int value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc1, 0x10]; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.is_int(), false); + /// assert_eq!(rlp.at(0).is_int(), true); + /// } + /// ``` + pub fn is_int(&self) -> bool { + self.rlp.is_int() + } + /// Get iterator over rlp-slices /// /// ```rust @@ -384,6 +401,32 @@ impl<'a> UntrustedRlp<'a> { !self.is_null() && self.bytes[0] < 0xc0 } + /// Int value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc1, 0x10]; + /// let rlp = UntrustedRlp::new(&data); + /// assert_eq!(rlp.is_int(), false); + /// assert_eq!(rlp.at(0).unwrap().is_int(), true); + /// } + /// ``` + pub fn is_int(&self) -> bool { + if self.is_null() { + return false; + } + + match self.bytes[0] { + 0...0x80 => true, + 0x81...0xb7 => self.bytes[1] != 0, + b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0, + _ => false + } + } + /// Get iterator over rlp-slices /// /// ```rust From b6dc3e464dc125b8b9eb80ed19d8f3aaf93247d6 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 12:41:11 +0100 Subject: [PATCH 128/379] rlp items functions --- src/rlp.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index ba1b5498b..78cd70d6d 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -152,6 +152,24 @@ impl<'a> Rlp<'a> { self.rlp.data() } + /// Returns number of rlp items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.items(), 2); + /// let view = rlp.at(1); + /// assert_eq!(view.items(), 0); + /// } + /// ``` + pub fn items(&self) -> usize { + self.rlp.items() + } + /// Get view onto rlp-slice at index. /// /// Caches offset to given index, so access to successive @@ -297,6 +315,27 @@ impl<'a> UntrustedRlp<'a> { self.bytes } + /// Returns number of rlp items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert_eq!(rlp.items(), 2); + /// let view = rlp.at(1).unwrap(); + /// assert_eq!(view.items(), 0); + /// } + /// ``` + pub fn items(&self) -> usize { + match self.is_list() { + true => self.iter().count(), + false => 0 + } + } + /// Get view onto rlp-slice at index /// /// Caches offset to given index, so access to successive From 0a93c7c32eba4cb042825f64406b0ca898aa3e7e Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 12:58:23 +0100 Subject: [PATCH 129/379] Rlp.items -> Rlp.item_count, RlpStream.append_null -> RlpStream.append_empty_data --- src/rlp.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 78cd70d6d..8454600b7 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -161,13 +161,13 @@ impl<'a> Rlp<'a> { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.items(), 2); + /// assert_eq!(rlp.item_count(), 2); /// let view = rlp.at(1); - /// assert_eq!(view.items(), 0); + /// assert_eq!(view.item_count(), 0); /// } /// ``` - pub fn items(&self) -> usize { - self.rlp.items() + pub fn item_count(&self) -> usize { + self.rlp.item_count() } /// Get view onto rlp-slice at index. @@ -324,12 +324,12 @@ impl<'a> UntrustedRlp<'a> { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.items(), 2); + /// assert_eq!(rlp.item_count(), 2); /// let view = rlp.at(1).unwrap(); - /// assert_eq!(view.items(), 0); + /// assert_eq!(view.item_count(), 0); /// } /// ``` - pub fn items(&self) -> usize { + pub fn item_count(&self) -> usize { match self.is_list() { true => self.iter().count(), false => 0 @@ -784,12 +784,12 @@ impl RlpStream { /// /// fn main () { /// let mut stream = RlpStream::new_list(2); - /// stream.append_null().append_null(); + /// stream.append_empty_data().append_empty_data(); /// let out = stream.out(); /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); /// } /// ``` - pub fn append_null<'a>(&'a mut self) -> &'a mut RlpStream { + pub fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { // self push raw item self.encoder.bytes.push(0x80); From be1d208c361eb9fad35e2e994ec11f72fec4157d Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 13:06:51 +0100 Subject: [PATCH 130/379] rlp size function --- src/rlp.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index 8454600b7..b9f58e78f 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -170,6 +170,24 @@ impl<'a> Rlp<'a> { self.rlp.item_count() } + /// Returns the number of bytes in the data, or zero if it isn't data. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.size(), 0); + /// let view = rlp.at(1); + /// assert_eq!(view.size(), 3); + /// } + /// ``` + pub fn size(&self) -> usize { + self.rlp.size() + } + /// Get view onto rlp-slice at index. /// /// Caches offset to given index, so access to successive @@ -336,6 +354,27 @@ impl<'a> UntrustedRlp<'a> { } } + /// Returns the number of bytes in the data, or zero if it isn't data. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = UntrustedRlp::new(&data); + /// assert_eq!(rlp.size(), 0); + /// let view = rlp.at(1).unwrap(); + /// assert_eq!(view.size(), 3); + /// } + /// ``` + pub fn size(&self) -> usize { + match self.is_data() { + true => Self::item_info(self.bytes).unwrap().value_len, + false => 0 + } + } + /// Get view onto rlp-slice at index /// /// Caches offset to given index, so access to successive From 2d556d6f06d25afa5a677e8ce37e1704105a7a70 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 30 Nov 2015 13:19:55 +0100 Subject: [PATCH 131/379] Add prototype. --- src/rlp.rs | 199 ++++++++++++++++++++++++++++------------------------ src/trie.rs | 13 ++-- 2 files changed, 116 insertions(+), 96 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index b9f58e78f..81355fb5d 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -126,6 +126,12 @@ impl<'a> From> for UntrustedRlp<'a> { } } +pub enum Prototype { + Null, + Data(usize), + List(usize), +} + impl<'a> Rlp<'a> { /// Create a new instance of `Rlp` pub fn new(bytes: &'a [u8]) -> Rlp<'a> { @@ -134,6 +140,19 @@ impl<'a> Rlp<'a> { } } + /// Get the prototype of the RLP. + pub fn prototype(&self) -> Prototype { + if self.is_data() { + Prototype::Data(self.size()) + } + else if self.is_list() { + Prototype::List(self.item_count()) + } + else { + Prototype::Null + } + } + /// The bare data of the rlp. /// /// ```rust @@ -148,7 +167,7 @@ impl<'a> Rlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&'a self) -> &'a [u8] { + pub fn data(&self) -> &[u8] { self.rlp.data() } @@ -588,7 +607,7 @@ pub struct UntrustedRlpIterator<'a> { impl<'a> IntoIterator for &'a UntrustedRlp<'a> { type Item = UntrustedRlp<'a>; - type IntoIter = UntrustedRlpIterator<'a>; + type IntoIter = UntrustedRlpIterator<'a>; fn into_iter(self) -> Self::IntoIter { UntrustedRlpIterator { @@ -617,7 +636,7 @@ pub struct RlpIterator<'a> { impl<'a> IntoIterator for &'a Rlp<'a> { type Item = Rlp<'a>; - type IntoIter = RlpIterator<'a>; + type IntoIter = RlpIterator<'a>; fn into_iter(self) -> Self::IntoIter { RlpIterator { @@ -1147,63 +1166,63 @@ mod tests { #[test] fn encode_u16() { let tests = vec![ - ETestPair(0u16, vec![0x80u8]), - ETestPair(0x100, vec![0x82, 0x01, 0x00]), - ETestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; + ETestPair(0u16, vec![0x80u8]), + ETestPair(0x100, vec![0x82, 0x01, 0x00]), + ETestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_u32() { let tests = vec![ - ETestPair(0u32, vec![0x80u8]), - ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; + ETestPair(0u32, vec![0x80u8]), + ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_u64() { let tests = vec![ - ETestPair(0u64, vec![0x80u8]), - ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; + ETestPair(0u64, vec![0x80u8]), + ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_u256() { let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), - ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; run_encode_tests(tests); } #[test] fn encode_str() { let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), - ETestPair("dog", vec![0x83, b'd', b'o', b'g']), - ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), - ETestPair("", vec![0x80]), - ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; + ETestPair("dog", vec![0x83, b'd', b'o', b'g']), + ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), + ETestPair("", vec![0x80]), + ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; run_encode_tests(tests); } @@ -1214,8 +1233,8 @@ mod tests { let tests = vec![ ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) ]; run_encode_tests(tests); } @@ -1235,18 +1254,18 @@ mod tests { #[test] fn encode_vector_u64() { let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u64], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; + ETestPair(vec![], vec![0xc0]), + ETestPair(vec![15u64], vec![0xc1, 0x0f]), + ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; run_encode_tests(tests); } #[test] fn encode_vector_str() { let tests = vec![ETestPair(vec!["cat", "dog"], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; run_encode_tests(tests); } @@ -1262,7 +1281,7 @@ mod tests { stream.append(&"cat").append(&"dog"); let out = stream.out(); assert_eq!(out, - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); } #[test] @@ -1284,7 +1303,7 @@ mod tests { } let out = stream.out(); assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); } @@ -1326,65 +1345,65 @@ mod tests { #[test] fn decode_untrusted_u16() { let tests = vec![ - DTestPair(0u16, vec![0u8]), - DTestPair(0x100, vec![0x82, 0x01, 0x00]), - DTestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; + DTestPair(0u16, vec![0u8]), + DTestPair(0x100, vec![0x82, 0x01, 0x00]), + DTestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; run_decode_tests(tests); } #[test] fn decode_untrusted_u32() { let tests = vec![ - DTestPair(0u32, vec![0u8]), - DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; + DTestPair(0u32, vec![0u8]), + DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; run_decode_tests(tests); } #[test] fn decode_untrusted_u64() { let tests = vec![ - DTestPair(0u64, vec![0u8]), - DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; + DTestPair(0u64, vec![0u8]), + DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; run_decode_tests(tests); } #[test] fn decode_untrusted_u256() { let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), - DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; run_decode_tests(tests); } #[test] fn decode_untrusted_str() { let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), - DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), - DTestPair("Marek".to_string(), - vec![0x85, b'M', b'a', b'r', b'e', b'k']), - DTestPair("".to_string(), vec![0x80]), - DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" - .to_string(), - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; + DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), + DTestPair("Marek".to_string(), + vec![0x85, b'M', b'a', b'r', b'e', b'k']), + DTestPair("".to_string(), vec![0x80]), + DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" + .to_string(), + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; run_decode_tests(tests); } @@ -1395,8 +1414,8 @@ mod tests { let tests = vec![ DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) ]; run_decode_tests(tests); } @@ -1404,25 +1423,25 @@ mod tests { #[test] fn decode_untrusted_vector_u64() { let tests = vec![ - DTestPair(vec![], vec![0xc0]), - DTestPair(vec![15u64], vec![0xc1, 0x0f]), - DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; + DTestPair(vec![], vec![0xc0]), + DTestPair(vec![15u64], vec![0xc1, 0x0f]), + DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; run_decode_tests(tests); } #[test] fn decode_untrusted_vector_str() { let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; run_decode_tests(tests); } #[test] fn decode_untrusted_vector_of_vectors_str() { let tests = vec![DTestPair(vec![vec!["cat".to_string()]], - vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; run_decode_tests(tests); } } diff --git a/src/trie.rs b/src/trie.rs index 80e753f84..9aaa83477 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -49,7 +49,7 @@ impl TrieDB { self.root = self.db.insert(root_data); } - fn insert(&mut self, key: &NibbleSlice, value: &[u8]) { + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: (Bytes, Diff); { @@ -76,7 +76,8 @@ impl TrieDB { /// The database will be updated so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. fn merge(&self, old: &[u8], partial_key: &NibbleSlice, value: &[u8]) -> (Bytes, Diff) { - let o = Rlp::new(old); + unimplemented!(); +/* let o = Rlp::new(old); match (o.type()) { List(17) => { // already have a branch. route and merge. @@ -85,8 +86,8 @@ impl TrieDB { // already have an extension. either fast_forward, cleve or transmute_to_branch. }, Data(0) => compose_extension(partial_key), - _ -> panic!("Invalid RLP for node."), - } + _ => panic!("Invalid RLP for node."), + }*/ } fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { @@ -94,7 +95,7 @@ impl TrieDB { s.append(&partial_key.encoded(is_leaf)); s.append(&value.to_vec()); // WTF?!?! //s.append(value); // <-- should be. - s.out().unwrap() + s.out() } } @@ -110,7 +111,7 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - (self as &mut TrieDB).insert(&NibbleSlice::new(key), value); + (self as &mut TrieDB).add(&NibbleSlice::new(key), value); } fn remove(&mut self, _key: &[u8]) { From 7fd62527727dafc56424b8131127c3bf00466cf8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 30 Nov 2015 13:25:37 +0100 Subject: [PATCH 132/379] Fix warnings, use Prototype. --- src/trie.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 9aaa83477..8dd7fce90 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -34,6 +34,10 @@ struct Diff { old: Vec, } +impl Diff { + pub fn new() -> Diff { Diff { new: vec![], old: vec![] }} +} + impl TrieDB { pub fn new(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } } @@ -51,11 +55,11 @@ impl TrieDB { fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. - let mut todo: (Bytes, Diff); + let todo = { let root_rlp = self.db.lookup(&self.root).unwrap(); - todo = self.merge(root_rlp, key, value); - } + self.merge(root_rlp, key, value) + }; self.apply(todo.1); self.set_root_rlp(&todo.0); } @@ -76,18 +80,19 @@ impl TrieDB { /// The database will be updated so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. fn merge(&self, old: &[u8], partial_key: &NibbleSlice, value: &[u8]) -> (Bytes, Diff) { - unimplemented!(); -/* let o = Rlp::new(old); - match (o.type()) { - List(17) => { + let o = Rlp::new(old); + match o.prototype() { + Prototype::List(17) => { // already have a branch. route and merge. + unimplemented!(); }, - List(2) => { + Prototype::List(2) => { // already have an extension. either fast_forward, cleve or transmute_to_branch. + unimplemented!(); }, - Data(0) => compose_extension(partial_key), + Prototype::Data(0) => (Self::compose_extension(partial_key, value, true), Diff::new()), _ => panic!("Invalid RLP for node."), - }*/ + } } fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { From 972b78d47bc159682f3b2603f0fec557dd395db3 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 30 Nov 2015 14:53:22 +0100 Subject: [PATCH 133/379] First insert in trie works! --- src/trie.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 8dd7fce90..13b0a4e9a 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -50,14 +50,15 @@ impl TrieDB { pub fn db(&self) -> &HashDB { self.db.as_ref() } fn set_root_rlp(&mut self, root_data: &[u8]) { + self.db.kill(&self.root); self.root = self.db.insert(root_data); + println!("set_root_rlp {:?} {:?}", root_data, self.root); } fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. - let todo = - { - let root_rlp = self.db.lookup(&self.root).unwrap(); + let todo = { + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); self.merge(root_rlp, key, value) }; self.apply(todo.1); @@ -90,17 +91,22 @@ impl TrieDB { // already have an extension. either fast_forward, cleve or transmute_to_branch. unimplemented!(); }, - Prototype::Data(0) => (Self::compose_extension(partial_key, value, true), Diff::new()), + Prototype::Data(0) => { + (Self::compose_extension(partial_key, value, true), Diff::new()) + }, _ => panic!("Invalid RLP for node."), } } fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { + println!("compose_extension {:?} {:?} {:?} ({:?})", partial_key, value, is_leaf, partial_key.encoded(is_leaf)); let mut s = RlpStream::new_list(2); s.append(&partial_key.encoded(is_leaf)); s.append(&value.to_vec()); // WTF?!?! //s.append(value); // <-- should be. - s.out() + let r = s.out(); + println!("output: -> {:?}", &r); + r } } @@ -127,6 +133,7 @@ impl Trie for TrieDB { #[test] fn playpen() { use overlaydb::*; + use triehash::*; (&[1, 2, 3]).starts_with(&[1, 2]); @@ -134,5 +141,7 @@ fn playpen() { t.init(); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(*t.root(), hash256(&[ NibblePair::new_raw(vec![0x01u8, 0x23], vec![0x01u8, 0x23])])); } \ No newline at end of file From ddd0baa70a9982a5e0d00f361065f5412d8c9c99 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 30 Nov 2015 15:01:43 +0100 Subject: [PATCH 134/379] data_at in RLP. --- src/rlp.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rlp.rs b/src/rlp.rs index 81355fb5d..c3f237092 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -227,6 +227,10 @@ impl<'a> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } + pub fn data_at(&self, _index: usize) -> &[u8] { + unimplemented!(); + } + /// No value /// /// ```rust From ec7ce060e49aa2814619131f184a47d68a3c955d Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 15:35:10 +0100 Subject: [PATCH 135/379] triehash cleanup in progress --- src/rlp.rs | 12 ++- src/triehash.rs | 269 +++++++++++++++++------------------------------- 2 files changed, 106 insertions(+), 175 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index b9f58e78f..59eb4dd6a 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -148,7 +148,7 @@ impl<'a> Rlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&'a self) -> &'a [u8] { + pub fn data(&self) -> &[u8] { self.rlp.data() } @@ -329,7 +329,7 @@ impl<'a> UntrustedRlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&'a self) -> &'a [u8] { + pub fn data(&self) -> &[u8] { self.bytes } @@ -1256,6 +1256,14 @@ mod tests { run_encode_tests(tests); } + #[test] + fn encode_bytes() { + let vec = vec![0u8]; + let slice: &[u8] = &vec; + let res = rlp::encode(&slice); + assert_eq!(res, vec![0u8]); + } + #[test] fn rlp_stream() { let mut stream = RlpStream::new_list(2); diff --git a/src/triehash.rs b/src/triehash.rs index aaeef5f65..155e7c8aa 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -8,6 +8,39 @@ use rlp; use rlp::RlpStream; use vector::SharedPrefix; +pub fn ordered_trie_root(input: Vec>) -> H256 { + let gen_input = input + // first put elements into btree to sort them by nibbles + // optimize it later + .into_iter() + .fold(BTreeMap::new(), | mut acc, vec | { + let len = acc.len(); + acc.insert(as_nibbles(&rlp::encode(&len)), vec); + acc + }) + // then move them to a vector + .into_iter() + .map(|p| p ) + .collect(); + + gen_trie_root(gen_input) +} + +pub fn trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let gen_input = input + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v)) + .collect(); + + gen_trie_root(gen_input) +} + +fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let mut stream = RlpStream::new(); + hash256rlp(&input, 0, &mut stream); + stream.out().sha3() +} + /// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. /// /// The "termination marker" and "leaf-node" specifier are completely equivalent. @@ -65,7 +98,7 @@ use vector::SharedPrefix; /// } /// ``` /// -pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { +fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { let inlen = nibbles.len(); let oddness_factor = inlen % 2; // next even number divided by two @@ -110,7 +143,7 @@ pub fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { /// assert_eq!(as_nibbles(&v), e); /// } /// ``` -pub fn as_nibbles(bytes: &[u8]) -> Vec { +fn as_nibbles(bytes: &[u8]) -> Vec { let mut res = vec![]; res.reserve(bytes.len() * 2); for i in 0..bytes.len() { @@ -120,98 +153,44 @@ pub fn as_nibbles(bytes: &[u8]) -> Vec { res } -#[derive(Debug)] -pub struct NibblePair { - nibble: Vec, - data: Vec -} - -impl NibblePair { - pub fn new(nibble: Vec, data: Vec) -> NibblePair { - NibblePair { - nibble: nibble, - data: data - } - } - - pub fn new_raw(to_nibble: Vec, data: Vec) -> NibblePair { - NibblePair::new(as_nibbles(&to_nibble), data) - } -} - -fn nibble_shared_prefix_len(vec: &[NibblePair]) -> usize { - if vec.len() == 0 { - return 0; - } - - vec.iter() - // skip first element - .skip(1) - // get minimum number of shared nibbles between first and each successive - .fold(vec[0].nibble.len(), | acc, pair | { - cmp::min(vec[0].nibble.shared_prefix_len(&pair.nibble), acc) - }) -} - -pub fn ordered_trie_root(data: Vec>) -> H256 { - let vec: Vec = data - // first put elements into btree to sort them by nibbles - // optimize it later - .into_iter() - .fold(BTreeMap::new(), | mut acc, vec | { - let len = acc.len(); - acc.insert(as_nibbles(&rlp::encode(&len)), vec); - acc - }) - // then move them to a vector - .into_iter() - .map(|(k, v)| NibblePair::new(k, v) ) - .collect(); - - hash256(&vec) -} - -pub fn hash256(vec: &[NibblePair]) -> H256 { - let out = match vec.len() { - 0 => rlp::encode(&""), - _ => { - let mut stream = RlpStream::new(); - hash256rlp(&vec, 0, &mut stream); - stream.out() - } - }; - - out.sha3() -} - -fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { - let vlen = vec.len(); +fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { + let inlen = input.len(); // in case of empty slice, just append null - if vlen == 0 { + if inlen == 0 { stream.append(&""); return; } + // take slices + let key: &Vec = &input[0].0; + let value: &[u8] = &input[0].1; + // if the slice contains just one item, append the suffix of the key // and then append value - if vlen == 1 { + if inlen == 1 { stream.append_list(2); - stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..], true)); - stream.append(&vec[0].data); + stream.append(&hex_prefix_encode(&key[pre_len..], true)); + stream.append(&value); return; } // get length of the longest shared prefix in slice keys - let shared_prefix = nibble_shared_prefix_len(vec); + let shared_prefix = input.iter() + // skip first element + .skip(1) + // get minimum number of shared nibbles between first and each successive + .fold(key.len(), | acc, &(ref k, _) | { + cmp::min(key.shared_prefix_len(&k), acc) + }); // if shared prefix is higher than current prefix append its // new part of the key to the stream // then recursively append suffixes of all items who had this key if shared_prefix > pre_len { stream.append_list(2); - stream.append(&hex_prefix_encode(&vec[0].nibble[pre_len..shared_prefix], false)); - hash256aux(vec, shared_prefix, stream); + stream.append(&hex_prefix_encode(&key[pre_len..shared_prefix], false)); + hash256aux(input, shared_prefix, stream); return; } @@ -220,7 +199,7 @@ fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { stream.append_list(17); // if first key len is equal to prefix_len, move to next element - let mut begin = match pre_len == vec[0].nibble.len() { + let mut begin = match pre_len == key.len() { true => 1, false => 0 }; @@ -228,29 +207,29 @@ fn hash256rlp(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { // iterate over all possible nibbles for i in 0..16 { // cout how many successive elements have same next nibble - let len = vec[begin..].iter() - .map(| pair | pair.nibble[pre_len] ) + let len = input[begin..].iter() + .map(| pair | pair.0[pre_len] ) .take_while(|&q| q == i).count(); // if at least 1 successive element has the same nibble // append their suffixes match len { 0 => { stream.append(&""); }, - _ => hash256aux(&vec[begin..(begin + len)], pre_len + 1, stream) + _ => hash256aux(&input[begin..(begin + len)], pre_len + 1, stream) } begin += len; } // if fist key len is equal prefix, append it's value - match pre_len == vec[0].nibble.len() { - true => { stream.append(&vec[0].data); }, + match pre_len == key.len() { + true => { stream.append(&value); }, false => { stream.append(&""); } }; } -fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { +fn hash256aux(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { let mut s = RlpStream::new(); - hash256rlp(vec, pre_len, &mut s); + hash256rlp(input, pre_len, &mut s); let out = s.out(); match out.len() { 0...31 => stream.append_raw(&out, 1), @@ -258,39 +237,6 @@ fn hash256aux(vec: &[NibblePair], pre_len: usize, stream: &mut RlpStream) { }; } - -#[test] -fn test_shared_nibble_len() { - let len = nibble_shared_prefix_len(&vec![ - NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), - NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), - ]); - - assert_eq!(len , 7); -} - -#[test] -fn test_shared_nibble_len2() { - let len = nibble_shared_prefix_len(&vec![ - NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), - NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), - NibblePair::new(vec![4, 1, 2, 3, 4, 5, 6], vec![]) - ]); - - assert_eq!(len , 0); -} - -#[test] -fn test_shared_nibble_len3() { - let len = nibble_shared_prefix_len(&vec![ - NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), - NibblePair::new(vec![0, 1, 2, 3, 4, 5, 6], vec![]), - NibblePair::new(vec![0, 1, 2, 4, 4, 5, 6], vec![]) - ]); - - assert_eq!(len , 3); -} - #[cfg(test)] mod tests { use std::str::FromStr; @@ -298,106 +244,83 @@ mod tests { use hash::*; use triehash::*; - - #[test] fn empty_trie_root() { - assert_eq!(hash256(&vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); + assert_eq!(trie_root(vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); } #[test] fn single_trie_item() { - - let v = vec![ - NibblePair::new_raw(From::from("A"), - From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")) - ]; - - assert_eq!(hash256(&v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); + let v = vec![(From::from("A"), From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))]; + assert_eq!(trie_root(v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); } #[test] fn foo_trie_item() { let v = vec![ - NibblePair::new_raw(From::from("foo"), - From::from("bar")), - NibblePair::new_raw(From::from("food"), - From::from("bass")) + (From::from("foo"), From::from("bar")), + (From::from("food"), From::from("bass")) ]; - assert_eq!(hash256(&v), H256::from_str("17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3").unwrap()); + assert_eq!(trie_root(v), H256::from_str("17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3").unwrap()); } #[test] fn dogs_trie_item() { let v = vec![ - NibblePair::new_raw(From::from("doe"), - From::from("reindeer")), - - NibblePair::new_raw(From::from("dog"), - From::from("puppy")), - - NibblePair::new_raw(From::from("dogglesworth"), - From::from("cat")), + (From::from("doe"), From::from("reindeer")), + (From::from("dog"), From::from("puppy")), + (From::from("dogglesworth"), From::from("cat")), ]; - assert_eq!(hash256(&v), H256::from_str("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3").unwrap()); + assert_eq!(trie_root(v), H256::from_str("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3").unwrap()); } #[test] fn puppy_trie_items() { let v = vec![ - NibblePair::new_raw(From::from("do"), - From::from("verb")), - - NibblePair::new_raw(From::from("dog"), - From::from("puppy")), - - NibblePair::new_raw(From::from("doge"), - From::from("coin")), - - NibblePair::new_raw(From::from("horse"), - From::from("stallion")), - + (From::from("do"), From::from("verb")), + (From::from("dog"), From::from("puppy")), + (From::from("doge"), From::from("coin")), + (From::from("horse"), From::from("stallion")), ]; - assert_eq!(hash256(&v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); + assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); } #[test] fn test_trie_root() { let v = vec![ - NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), - "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), + + ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), - NibblePair::new_raw("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), - "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), + "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - NibblePair::new_raw("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), - "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), + ("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), + "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), - NibblePair::new_raw("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), - "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + ("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), + "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - NibblePair::new_raw("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), - "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), + ("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), + "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), - NibblePair::new_raw("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), - "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), + ("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), - NibblePair::new_raw("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), - "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + ("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + + ("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), + "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) - NibblePair::new_raw("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), - "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) ]; - let root = H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap(); - - let res = hash256(&v); - assert_eq!(res, root); + assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); } } From 4a912870d1dbd5fc5fd0151fce859bd5b8188a3f Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 15:56:41 +0100 Subject: [PATCH 136/379] docs for triehash --- src/triehash.rs | 138 +++++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 55 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 155e7c8aa..bc8dedf76 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,4 +1,6 @@ -//! Generete trie root +//! Generete trie root. +//! +//! This module should be used to generate trie root hash. use std::collections::BTreeMap; use std::cmp; @@ -8,6 +10,21 @@ use rlp; use rlp::RlpStream; use vector::SharedPrefix; +// todo: verify if example for ordered_trie_root is valid +/// Generates a trie root hash for a vector of values +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::triehash::*; +/// use util::hash::*; +/// +/// fn main() { +/// let v = vec![From::from("doe"), From::from("reindeer")]; +/// let root = "e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3"; +/// assert_eq!(ordered_trie_root(v), H256::from_str(root).unwrap()); +/// } +/// ``` pub fn ordered_trie_root(input: Vec>) -> H256 { let gen_input = input // first put elements into btree to sort them by nibbles @@ -26,6 +43,25 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { gen_trie_root(gen_input) } +/// Generates a trie root hash for a vector of key-values +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::triehash::*; +/// use util::hash::*; +/// +/// fn main() { +/// let v = vec![ +/// (From::from("doe"), From::from("reindeer")), +/// (From::from("dog"), From::from("puppy")), +/// (From::from("dogglesworth"), From::from("cat")), +/// ]; +/// +/// let root = "8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"; +/// assert_eq!(trie_root(v), H256::from_str(root).unwrap()); +/// } +/// ``` pub fn trie_root(input: Vec<(Vec, Vec)>) -> H256 { let gen_input = input .into_iter() @@ -60,44 +96,6 @@ fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { /// [1,2,3,4,5,T] 0x312345 // 5 > 3 /// [1,2,3,4,T] 0x201234 // 4 > 3 /// ``` -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::triehash::*; -/// -/// fn main() { -/// let v = vec![0, 0, 1, 2, 3, 4, 5]; -/// let e = vec![0x10, 0x01, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, false); -/// assert_eq!(h, e); -/// -/// let v = vec![0, 1, 2, 3, 4, 5]; -/// let e = vec![0x00, 0x01, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, false); -/// assert_eq!(h, e); -/// -/// let v = vec![0, 1, 2, 3, 4, 5]; -/// let e = vec![0x20, 0x01, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, true); -/// assert_eq!(h, e); -/// -/// let v = vec![1, 2, 3, 4, 5]; -/// let e = vec![0x31, 0x23, 0x45]; -/// let h = hex_prefix_encode(&v, true); -/// assert_eq!(h, e); -/// -/// let v = vec![1, 2, 3, 4]; -/// let e = vec![0x00, 0x12, 0x34]; -/// let h = hex_prefix_encode(&v, false); -/// assert_eq!(h, e); -/// -/// let v = vec![4, 1]; -/// let e = vec![0x20, 0x41]; -/// let h = hex_prefix_encode(&v, true); -/// assert_eq!(h, e); -/// } -/// ``` -/// fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { let inlen = nibbles.len(); let oddness_factor = inlen % 2; @@ -127,22 +125,6 @@ fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { } /// Converts slice of bytes to nibbles. -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::triehash::*; -/// -/// fn main () { -/// let v = vec![0x31, 0x23, 0x45]; -/// let e = vec![3, 1, 2, 3, 4, 5]; -/// assert_eq!(as_nibbles(&v), e); -/// -/// // A => 65 => 0x41 => [4, 1] -/// let v: Vec = From::from("A"); -/// let e = vec![4, 1]; -/// assert_eq!(as_nibbles(&v), e); -/// } -/// ``` fn as_nibbles(bytes: &[u8]) -> Vec { let mut res = vec![]; res.reserve(bytes.len() * 2); @@ -237,6 +219,52 @@ fn hash256aux(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre }; } + +#[test] +fn test_nibbles() { + let v = vec![0x31, 0x23, 0x45]; + let e = vec![3, 1, 2, 3, 4, 5]; + assert_eq!(as_nibbles(&v), e); + + // A => 65 => 0x41 => [4, 1] + let v: Vec = From::from("A"); + let e = vec![4, 1]; + assert_eq!(as_nibbles(&v), e); +} + +#[test] +fn test_hex_prefix_encode() { + let v = vec![0, 0, 1, 2, 3, 4, 5]; + let e = vec![0x10, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x00, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![0, 1, 2, 3, 4, 5]; + let e = vec![0x20, 0x01, 0x23, 0x45]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4, 5]; + let e = vec![0x31, 0x23, 0x45]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); + + let v = vec![1, 2, 3, 4]; + let e = vec![0x00, 0x12, 0x34]; + let h = hex_prefix_encode(&v, false); + assert_eq!(h, e); + + let v = vec![4, 1]; + let e = vec![0x20, 0x41]; + let h = hex_prefix_encode(&v, true); + assert_eq!(h, e); +} + #[cfg(test)] mod tests { use std::str::FromStr; From 43326356d074c6258d5356ef53c0a7c97bb70f6b Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 16:01:47 +0100 Subject: [PATCH 137/379] updated trie tests --- src/trie.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 13b0a4e9a..26eaafb2b 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -143,5 +143,7 @@ fn playpen() { assert!(t.is_empty()); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), hash256(&[ NibblePair::new_raw(vec![0x01u8, 0x23], vec![0x01u8, 0x23])])); -} \ No newline at end of file + assert_eq!(*t.root(), trie_root(vec![ + (vec![1u8, 0x23], vec![1u8, 0x23]) + ])); +} From d293bce22dc5fd551baef0417fd21b85fa258f8d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 30 Nov 2015 16:06:29 +0100 Subject: [PATCH 138/379] Trie progress... --- src/nibbleslice.rs | 14 ++++++++++++++ src/rlp.rs | 4 ---- src/trie.rs | 15 ++++++++++++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 5866ed8f0..6ccf695c0 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -38,6 +38,11 @@ impl<'a> NibbleSlice<'a> { /// Create a new nibble slice with the given byte-slice with a nibble offset. pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset} } + /// Create a new nibble slice from the given HPE encoded data (e.g. output of `encoded()`). + pub fn from_encoded(data: &'a [u8]) -> (NibbleSlice, bool) { + (Self::new_offset(data, if data[0] & 16 == 16 {1} else {2}), data[0] & 32 == 32) + } + /// Is this an empty slice? pub fn is_empty(&self) -> bool { self.len() == 0 } @@ -148,6 +153,15 @@ mod tests { assert_eq!(n.mid(1).encoded(true), &[0x31, 0x23, 0x45]); } + #[test] + fn from_encoded() { + let n = NibbleSlice::new(D); + assert_eq!((n, false), NibbleSlice::from_encoded(&[0x00, 0x01, 0x23, 0x45])); + assert_eq!((n, true), NibbleSlice::from_encoded(&[0x20, 0x01, 0x23, 0x45])); + assert_eq!((n.mid(1), false), NibbleSlice::from_encoded(&[0x11, 0x23, 0x45])); + assert_eq!((n.mid(1), true), NibbleSlice::from_encoded(&[0x31, 0x23, 0x45])); + } + #[test] fn shared() { let n = NibbleSlice::new(D); diff --git a/src/rlp.rs b/src/rlp.rs index c3f237092..81355fb5d 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -227,10 +227,6 @@ impl<'a> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } - pub fn data_at(&self, _index: usize) -> &[u8] { - unimplemented!(); - } - /// No value /// /// ```rust diff --git a/src/trie.rs b/src/trie.rs index 13b0a4e9a..c204ce590 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -88,6 +88,19 @@ impl TrieDB { unimplemented!(); }, Prototype::List(2) => { + let their_key_rlp = o.at(0); + let (them, _) = NibbleSlice::from_encoded(o.data()); + match partial_key.common_prefix(&them) { + 0 => { + // transmute to branch here + }, + cp if cp == them.len() => { + // fast-forward + }, + cp => { + // cleve into two + branch in the middle + }, + } // already have an extension. either fast_forward, cleve or transmute_to_branch. unimplemented!(); }, @@ -143,5 +156,5 @@ fn playpen() { assert!(t.is_empty()); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), hash256(&[ NibblePair::new_raw(vec![0x01u8, 0x23], vec![0x01u8, 0x23])])); + assert_eq!(*t.root(), hash256(&[NibblePair::new_raw(vec![0x01u8, 0x23], vec![0x01u8, 0x23])])); } \ No newline at end of file From 6660b3d65feaff95c886ec09ffed5791e8a88e4b Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 16:14:03 +0100 Subject: [PATCH 139/379] replaced usages of append(&"") to append_empty_data() --- src/triehash.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index bc8dedf76..e26cc255f 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -138,9 +138,9 @@ fn as_nibbles(bytes: &[u8]) -> Vec { fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { let inlen = input.len(); - // in case of empty slice, just append null + // in case of empty slice, just append empty data if inlen == 0 { - stream.append(&""); + stream.append_empty_data(); return; } @@ -196,7 +196,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // if at least 1 successive element has the same nibble // append their suffixes match len { - 0 => { stream.append(&""); }, + 0 => { stream.append_empty_data(); }, _ => hash256aux(&input[begin..(begin + len)], pre_len + 1, stream) } begin += len; @@ -205,7 +205,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // if fist key len is equal prefix, append it's value match pre_len == key.len() { true => { stream.append(&value); }, - false => { stream.append(&""); } + false => { stream.append_empty_data(); } }; } From 4873c33fefa8ec7f116e2955ff9273f775e536b7 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 30 Nov 2015 16:38:55 +0100 Subject: [PATCH 140/379] Encryption primitives; Continue work on handshake --- Cargo.toml | 2 +- src/crypto.rs | 182 +++++++++++++++++++++++++------- src/hash.rs | 34 +++++- src/lib.rs | 2 +- src/network/connection.rs | 109 ++++++++++++++++++++ src/network/handshake.rs | 145 ++++++++++++++++++++++++++ src/network/host.rs | 211 ++++++++++++++++++++------------------ src/network/mod.rs | 22 ++++ src/sha3.rs | 4 + 9 files changed, 571 insertions(+), 140 deletions(-) create mode 100644 src/network/connection.rs create mode 100644 src/network/handshake.rs diff --git a/Cargo.toml b/Cargo.toml index 51e9f4470..170ac2990 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,5 @@ tiny-keccak = "0.3" rocksdb = "0.2.1" num = "0.1" lazy_static = "0.1.*" -secp256k1 = "0.5.1" +eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } rust-crypto = "0.2.34" diff --git a/src/crypto.rs b/src/crypto.rs index 0d1ae9c05..6843a204b 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -5,7 +5,8 @@ use rand::os::OsRng; pub type Secret=H256; pub type Public=H512; -pub type Signature=H520; + +pub use ::sha3::Hashable; #[derive(Debug)] pub enum CryptoError { @@ -94,47 +95,152 @@ impl KeyPair { } } -/// Recovers Public key from signed message hash. -pub fn recover(signature: &Signature, message: &H256) -> Result { - use secp256k1::*; - let context = Secp256k1::new(); - let rsig = try!(RecoverableSignature::from_compact(&context, &signature[0..64], try!(RecoveryId::from_i32(signature[64] as i32)))); - let publ = try!(context.recover(&try!(Message::from_slice(&message)), &rsig)); - let serialized = publ.serialize_vec(&context, false); - let p: Public = Public::from_slice(&serialized[1..65]); - Ok(p) -} -/// Returns siganture of message hash. -pub fn sign(secret: &Secret, message: &H256) -> Result { - use secp256k1::*; - let context = Secp256k1::new(); - let sec: &key::SecretKey = unsafe { ::std::mem::transmute(secret) }; - let s = try!(context.sign_recoverable(&try!(Message::from_slice(&message)), sec)); - let (rec_id, data) = s.serialize_compact(&context); - let mut signature: ::crypto::Signature = unsafe { ::std::mem::uninitialized() }; - signature.clone_from_slice(&data); - signature[64] = rec_id.to_i32() as u8; - Ok(signature) -} -/// Verify signature. -pub fn verify(public: &Public, signature: &Signature, message: &H256) -> Result { - use secp256k1::*; - let context = Secp256k1::new(); - let rsig = try!(RecoverableSignature::from_compact(&context, &signature[0..64], try!(RecoveryId::from_i32(signature[64] as i32)))); - let sig = rsig.to_standard(&context); +pub mod ec { + use hash::*; + use crypto::*; - let mut pdata: [u8; 65] = [4u8; 65]; - let ptr = pdata[1..].as_mut_ptr(); - let src = public.as_ptr(); - unsafe { ::std::ptr::copy_nonoverlapping(src, ptr, 64) }; - let publ = try!(key::PublicKey::from_slice(&context, &pdata)); - match context.verify(&try!(Message::from_slice(&message)), &sig, &publ) { - Ok(_) => Ok(true), - Err(Error::IncorrectSignature) => Ok(false), - Err(x) => Err(>::from(x)) + pub type Signature = H520; + /// Recovers Public key from signed message hash. + pub fn recover(signature: &Signature, message: &H256) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let rsig = try!(RecoverableSignature::from_compact(&context, &signature[0..64], try!(RecoveryId::from_i32(signature[64] as i32)))); + let publ = try!(context.recover(&try!(Message::from_slice(&message)), &rsig)); + let serialized = publ.serialize_vec(&context, false); + let p: Public = Public::from_slice(&serialized[1..65]); + Ok(p) + } + /// Returns siganture of message hash. + pub fn sign(secret: &Secret, message: &H256) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let sec: &key::SecretKey = unsafe { ::std::mem::transmute(secret) }; + let s = try!(context.sign_recoverable(&try!(Message::from_slice(&message)), sec)); + let (rec_id, data) = s.serialize_compact(&context); + let mut signature: ec::Signature = unsafe { ::std::mem::uninitialized() }; + signature.clone_from_slice(&data); + signature[64] = rec_id.to_i32() as u8; + Ok(signature) + } + /// Verify signature. + pub fn verify(public: &Public, signature: &Signature, message: &H256) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let rsig = try!(RecoverableSignature::from_compact(&context, &signature[0..64], try!(RecoveryId::from_i32(signature[64] as i32)))); + let sig = rsig.to_standard(&context); + + let mut pdata: [u8; 65] = [4u8; 65]; + let ptr = pdata[1..].as_mut_ptr(); + let src = public.as_ptr(); + unsafe { ::std::ptr::copy_nonoverlapping(src, ptr, 64) }; + let publ = try!(key::PublicKey::from_slice(&context, &pdata)); + match context.verify(&try!(Message::from_slice(&message)), &sig, &publ) { + Ok(_) => Ok(true), + Err(Error::IncorrectSignature) => Ok(false), + Err(x) => Err(>::from(x)) + } } } +pub mod ecdh { + use crypto::*; + + pub fn agree(secret: &Secret, public: &Public, ) -> Result { + use secp256k1::*; + let context = Secp256k1::new(); + let mut pdata: [u8; 65] = [4u8; 65]; + let ptr = pdata[1..].as_mut_ptr(); + let src = public.as_ptr(); + unsafe { ::std::ptr::copy_nonoverlapping(src, ptr, 64) }; + let publ = try!(key::PublicKey::from_slice(&context, &pdata)); + let sec: &key::SecretKey = unsafe { ::std::mem::transmute(secret) }; + let shared = ecdh::SharedSecret::new_raw(&context, &publ, &sec); + let s: Secret = unsafe { ::std::mem::transmute(shared) }; + Ok(s) + } +} + +pub mod ecies { + use hash::*; + use bytes::*; + use crypto::*; + + pub fn encrypt(public: &Public, plain: &[u8]) -> Result { + use ::rcrypto::digest::Digest; + use ::rcrypto::sha2::Sha256; + use ::rcrypto::hmac::Hmac; + use ::rcrypto::mac::Mac; + let r = try!(KeyPair::create()); + let z = try!(ecdh::agree(r.secret(), public)); + let mut key = [0u8; 32]; + let mut mkey = [0u8; 32]; + kdf(&z, &[0u8; 0], &mut key); + let mut hasher = Sha256::new(); + let mkey_material = &key[16..32]; + hasher.input(mkey_material); + hasher.result(&mut mkey); + let ekey = &key[0..16]; + + let mut msg = vec![0u8; (1 + 64 + 16 + plain.len() + 32)]; + msg[0] = 0x04u8; + { + let msgd = &mut msg[1..]; + r.public().copy_to(&mut msgd[0..64]); + { + let cipher = &mut msgd[(64 + 16)..(64 + 16 + plain.len())]; + aes::encrypt(ekey, &H128::new(), plain, cipher); + } + let mut hmac = Hmac::new(Sha256::new(), &mkey); + { + let cipher_iv = &msgd[64..(64 + 16 + plain.len())]; + hmac.input(cipher_iv); + } + hmac.raw_result(&mut msgd[(64 + 16 + plain.len())..]); + } + Ok(msg) + } + + fn kdf(secret: &Secret, s1: &[u8], dest: &mut [u8]) { + use ::rcrypto::digest::Digest; + use ::rcrypto::sha2::Sha256; + let mut hasher = Sha256::new(); + // SEC/ISO/Shoup specify counter size SHOULD be equivalent + // to size of hash output, however, it also notes that + // the 4 bytes is okay. NIST specifies 4 bytes. + let mut ctr = 1u32; + let mut written = 0usize; + while written < dest.len() { + let ctrs = [(ctr >> 24) as u8, (ctr >> 16) as u8, (ctr >> 8) as u8, ctr as u8]; + hasher.input(&ctrs); + hasher.input(secret); + hasher.input(s1); + hasher.result(&mut dest[written..(written + 32)]); + hasher.reset(); + written += 32; + ctr += 1; + } + } +} + +pub mod aes { + use hash::*; + use ::rcrypto::blockmodes::*; + use ::rcrypto::aessafe::*; + use ::rcrypto::symmetriccipher::*; + use ::rcrypto::buffer::*; + + pub fn encrypt(k: &[u8], iv: &H128, plain: &[u8], dest: &mut [u8]) { + let mut encryptor = CtrMode::new(AesSafe128Encryptor::new(k), iv[..].to_vec()); + encryptor.encrypt(&mut RefReadBuffer::new(plain), &mut RefWriteBuffer::new(dest), true).expect("Invalid length or padding"); + } + + pub fn decrypt(k: &[u8], iv: &H128, encrypted: &[u8], dest: &mut [u8]) { + let mut encryptor = CtrMode::new(AesSafe128Encryptor::new(k), iv[..].to_vec()); + encryptor.decrypt(&mut RefReadBuffer::new(encrypted), &mut RefWriteBuffer::new(dest), true).expect("Invalid length or padding"); + } +} + + #[cfg(test)] mod tests { diff --git a/src/hash.rs b/src/hash.rs index ad918ef3d..4f27ae566 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -2,7 +2,7 @@ use std::str::FromStr; use std::fmt; use std::ops; use std::hash::{Hash, Hasher}; -use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitAnd}; +use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitAnd, BitXor}; use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; @@ -19,6 +19,7 @@ pub trait FixedHash: Sized + BytesConvertable { fn mut_bytes(&mut self) -> &mut [u8]; fn from_slice(src: &[u8]) -> Self; fn clone_from_slice(&mut self, src: &[u8]) -> usize; + fn copy_to(&self, dest: &mut [u8]); fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; fn bloom_part(&self, m: usize) -> T where T: FixedHash; fn contains_bloom(&self, b: &T) -> bool where T: FixedHash; @@ -95,6 +96,13 @@ macro_rules! impl_hash { r } + fn copy_to(&self, dest: &mut[u8]) { + unsafe { + let min = ::std::cmp::min($size, dest.len()); + ::std::ptr::copy(self.0.as_ptr(), dest.as_mut_ptr(), min); + } + } + fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash { let bp: Self = b.bloom_part($size); let new_self = &bp | self; @@ -299,6 +307,30 @@ macro_rules! impl_hash { } } + /// BitXor on references + impl <'a> BitXor for &'a $from { + type Output = $from; + + fn bitxor(self, rhs: Self) -> Self::Output { + unsafe { + use std::mem; + let mut ret: $from = mem::uninitialized(); + for i in 0..$size { + ret.0[i] = self.0[i] ^ rhs.0[i]; + } + ret + } + } + } + + /// Moving BitXor + impl BitXor for $from { + type Output = $from; + + fn bitxor(self, rhs: Self) -> Self::Output { + &self ^ &rhs + } + } impl $from { pub fn hex(&self) -> String { format!("{}", self) diff --git a/src/lib.rs b/src/lib.rs index 27c927b2c..b39558d1b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,7 @@ pub mod math; pub mod chainfilter; pub mod crypto; -//pub mod network; +pub mod network; // reexports pub use std::str::FromStr; diff --git a/src/network/connection.rs b/src/network/connection.rs new file mode 100644 index 000000000..c5deddf58 --- /dev/null +++ b/src/network/connection.rs @@ -0,0 +1,109 @@ +use std::io::{self, Cursor, Read}; +use mio::*; +use mio::tcp::*; +use hash::*; +use bytes::*; +use network::host::Host; + +pub struct Connection { + pub token: Token, + pub socket: TcpStream, + rec_buf: Bytes, + rec_size: usize, + send_buf: Cursor, + interest: EventSet, +} + +pub enum WriteStatus { + Ongoing, + Complete +} + +impl Connection { + pub fn new(token: Token, socket: TcpStream) -> Connection { + Connection { + token: token, + socket: socket, + send_buf: Cursor::new(Bytes::new()), + rec_buf: Bytes::new(), + rec_size: 0, + interest: EventSet::hup(), + } + } + + pub fn expect(&mut self, size: usize) { + if self.rec_size != self.rec_buf.len() { + warn!(target:"net", "Unexpected connection read start"); + } + unsafe { self.rec_buf.set_len(size) } + self.rec_size = size; + } + + pub fn readable(&mut self) -> io::Result> { + if self.rec_size == 0 || self.rec_buf.len() >= self.rec_size { + warn!(target:"net", "Unexpected connection read"); + } + let max = self.rec_size - self.rec_buf.len(); + // resolve "multiple applicable items in scope [E0034]" error + let sock_ref = ::by_ref(&mut self.socket); + match sock_ref.take(max as u64).try_read_buf(&mut self.rec_buf) { + Ok(Some(_)) if self.rec_buf.len() == self.rec_size => Ok(Some(&self.rec_buf[0..self.rec_size])), + Ok(_) => Ok(None), + Err(e) => Err(e), + } + } + + pub fn send(&mut self, data: &[u8]) { //TODO: take ownership version + let send_size = self.send_buf.get_ref().len(); + if send_size != 0 || self.send_buf.position() as usize >= send_size { + warn!(target:"net", "Unexpected connection send start"); + } + if self.send_buf.get_ref().capacity() < data.len() { + let capacity = self.send_buf.get_ref().capacity(); + self.send_buf.get_mut().reserve(data.len() - capacity); + } + unsafe { self.send_buf.get_mut().set_len(data.len()) } + unsafe { ::std::ptr::copy_nonoverlapping(data.as_ptr(), self.send_buf.get_mut()[..].as_mut_ptr(), data.len()) }; + if !self.interest.is_writable() { + self.interest.insert(EventSet::writable()); + } + } + + pub fn writable(&mut self) -> io::Result { + let send_size = self.send_buf.get_ref().len(); + if (self.send_buf.position() as usize) >= send_size { + warn!(target:"net", "Unexpected connection data"); + return Ok(WriteStatus::Complete) + } + match self.socket.try_write_buf(&mut self.send_buf) { + Ok(_) if (self.send_buf.position() as usize) < send_size => { + self.interest.insert(EventSet::writable()); + Ok(WriteStatus::Ongoing) + }, + Ok(_) if (self.send_buf.position() as usize) == send_size => { + self.interest.remove(EventSet::writable()); + Ok(WriteStatus::Complete) + }, + Ok(_) => { panic!("Wrote past buffer");}, + Err(e) => Err(e) + } + } + + pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + trace!(target: "net", "connection register; token={:?}", self.token); + self.interest.insert(EventSet::readable()); + event_loop.register_opt(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + error!("Failed to reregister {:?}, {:?}", self.token, e); + Err(e) + }) + } + + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + trace!(target: "net", "connection reregister; token={:?}", self.token); + event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + error!("Failed to reregister {:?}, {:?}", self.token, e); + Err(e) + }) + } +} + diff --git a/src/network/handshake.rs b/src/network/handshake.rs new file mode 100644 index 000000000..ef1cf418b --- /dev/null +++ b/src/network/handshake.rs @@ -0,0 +1,145 @@ +use mio::*; +use mio::tcp::*; +use hash::*; +use crypto::*; +use crypto; +use network::connection::{Connection, WriteStatus}; +use network::host::{NodeId, Host, HostInfo}; +use network::Error; + +#[derive(PartialEq, Eq)] +enum HandshakeState { + New, + ReadingAuth, + WritingAuth, + ReadingAck, + WritingAck, + WritingHello, + ReadingHello, + StartSession, +} + +pub struct Handshake { + pub id: NodeId, + pub connection: Connection, + state: HandshakeState, + idle_timeout: Option, + ecdhe: KeyPair, + nonce: H256, + remote_public: Public, + remote_nonce: H256 +} + +impl Handshake { + pub fn new(token: Token, id: &NodeId, socket: TcpStream, nonce: &H256) -> Result { + Ok(Handshake { + id: id.clone(), + connection: Connection::new(token, socket), + state: HandshakeState::New, + idle_timeout: None, + ecdhe: try!(KeyPair::create()), + nonce: nonce.clone(), + remote_public: Public::new(), + remote_nonce: H256::new() + }) + } + + pub fn start(&mut self, host: &HostInfo, originated: bool) { + if originated { + self.write_auth(host); + } + else { + self.read_auth(); + }; + } + + pub fn done(&self) -> bool { + self.state == HandshakeState::StartSession + } + + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + Ok(()) + } + + pub fn writable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + match self.state { + HandshakeState::WritingAuth => { + match (try!(self.connection.writable())) { + WriteStatus::Complete => { try!(self.read_ack()); }, + _ => {} + }; + try!(self.connection.reregister(event_loop)); + }, + HandshakeState::WritingAck => { + match (try!(self.connection.writable())) { + WriteStatus::Complete => { try!(self.read_hello()); }, + _ => {} + }; + try!(self.connection.reregister(event_loop)); + }, + HandshakeState::WritingHello => { + match (try!(self.connection.writable())) { + WriteStatus::Complete => { self.state = HandshakeState::StartSession; }, + _ => { try!(self.connection.reregister(event_loop)); } + }; + }, + _ => { panic!("Unexpected state") } + } + Ok(()) + } + + pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); + self.connection.register(event_loop); + Ok(()) + } + + fn read_auth(&mut self) -> Result<(), Error> { + Ok(()) + } + + fn read_ack(&mut self) -> Result<(), Error> { + Ok(()) + } + + fn read_hello(&mut self) -> Result<(), Error> { + Ok(()) + } + + fn write_auth(&mut self, host: &HostInfo) -> Result<(), Error> { + trace!(target:"net", "Sending auth to {:?}", self.connection.socket.peer_addr()); + let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants + let len = data.len(); + { + data[len - 1] = 0x0; + let (sig, rest) = data.split_at_mut(65); + let (hepubk, rest) = rest.split_at_mut(32); + let (mut pubk, rest) = rest.split_at_mut(64); + let (nonce, rest) = rest.split_at_mut(32); + + // E(remote-pubk, S(ecdhe-random, ecdh-shared-secret^nonce) || H(ecdhe-random-pubk) || pubk || nonce || 0x0) + let shared = try!(crypto::ecdh::agree(host.secret(), &self.id)); + let signature = try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig); + self.ecdhe.public().sha3_into(hepubk); + host.id().copy_to(&mut pubk); + self.nonce.copy_to(nonce); + } + let message = try!(crypto::ecies::encrypt(&self.id, &data)); + self.connection.send(&message[..]); + self.state = HandshakeState::WritingAuth; + Ok(()) + } + + fn write_ack(&mut self) -> Result<(), Error> { + Ok(()) + } + + fn write_hello(&mut self) -> Result<(), Error> { + Ok(()) + } + + +} diff --git a/src/network/host.rs b/src/network/host.rs index c945b0e57..d436fae5f 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -11,9 +11,11 @@ use mio::util::{Slab}; use mio::tcp::*; use mio::udp::*; use hash::*; -use bytes::*; +use crypto::*; use time::Tm; use error::EthcoreError; +use network::connection::Connection; +use network::handshake::Handshake; const DEFAULT_PORT: u16 = 30303; @@ -27,9 +29,7 @@ const IDEAL_PEERS:u32 = 10; const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. -type NodeId = H512; -type PublicKey = H512; -type SecretKey = H256; +pub type NodeId = H512; #[derive(Debug)] struct NetworkConfiguration { @@ -47,7 +47,7 @@ impl NetworkConfiguration { public_address: SocketAddr::from_str("0.0.0.0:30303").unwrap(), no_nat: false, no_discovery: false, - pin: false + pin: false, } } } @@ -164,45 +164,6 @@ impl NodeBucket { } } -struct Connection { - socket: TcpStream, - send_queue: Vec, -} - -impl Connection { - fn new(socket: TcpStream) -> Connection { - Connection { - socket: socket, - send_queue: Vec::new(), - } - } -} - -#[derive(PartialEq, Eq)] -enum HandshakeState { - New, - AckAuth, - WriteHello, - ReadHello, - StartSession, -} - -struct Handshake { - id: NodeId, - connection: Connection, - state: HandshakeState, -} - -impl Handshake { - fn new(id: NodeId, socket: TcpStream) -> Handshake { - Handshake { - id: id, - connection: Connection::new(socket), - state: HandshakeState::New - } - } -} - struct Peer { id: NodeId, connection: Connection, @@ -214,7 +175,7 @@ impl FindNodePacket { fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { FindNodePacket } - fn sign(&mut self, _secret: &SecretKey) { + fn sign(&mut self, _secret: &Secret) { } fn send(& self, _socket: &mut UdpSocket) { @@ -236,11 +197,29 @@ pub enum HostMessage { Shutdown } -pub struct Host { - secret: SecretKey, - node: Node, - sender: Sender, +pub struct HostInfo { + keys: KeyPair, config: NetworkConfiguration, + nonce: H256 +} + +impl HostInfo { + pub fn id(&self) -> &NodeId { + self.keys.public() + } + + pub fn secret(&self) -> &Secret { + self.keys.secret() + } + pub fn next_nonce(&mut self) -> H256 { + self.nonce = self.nonce.sha3(); + return self.nonce.clone(); + } +} + +pub struct Host { + info: HostInfo, + sender: Sender, udp_socket: UdpSocket, listener: TcpListener, peers: Slab, @@ -282,9 +261,11 @@ impl Host { event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); let mut host = Host { - secret: SecretKey::new(), - node: Node::new(NodeId::new(), config.public_address.clone(), PeerType::Required), - config: config, + info: HostInfo { + keys: KeyPair::create().unwrap(), + config: config, + nonce: H256::random() + }, sender: sender, udp_socket: udp_socket, listener: listener, @@ -338,7 +319,7 @@ impl Host { } let mut tried_count = 0; { - let nearest = Host::nearest_node_entries(&self.node.id, &self.discovery_id, &self.node_buckets).into_iter(); + let nearest = Host::nearest_node_entries(&self.info.id(), &self.discovery_id, &self.node_buckets).into_iter(); let nodes = RefCell::new(&mut self.discovery_nodes); let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); for r in nearest { @@ -380,14 +361,14 @@ impl Host { ret } - fn nearest_node_entries<'a>(source: &NodeId, target: &NodeId, buckets: &'a Vec) -> Vec<&'a NodeId> + fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b Vec) -> Vec<&'b NodeId> { // send ALPHA FindNode packets to nodes we know, closest to target const LAST_BIN: u32 = NODE_BINS - 1; let mut head = Host::distance(source, target); let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; - let mut found: BTreeMap> = BTreeMap::new(); + let mut found: BTreeMap> = BTreeMap::new(); let mut count = 0; // if d is 0, then we roll look forward, if last, we reverse, else, spread from d @@ -463,7 +444,6 @@ impl Host { } fn maintain_network(&mut self, event_loop: &mut EventLoop) { - self.keep_alive(); self.connect_peers(event_loop); } @@ -491,7 +471,7 @@ impl Host { if connected && required { req_conn += 1; } - else if !connected && (!self.config.pin || required) { + else if !connected && (!self.info.config.pin || required) { to_connect.push(n); } } @@ -505,7 +485,7 @@ impl Host { } } - if !self.config.pin + if !self.info.config.pin { let pending_count = 0; //TODO: let peer_count = 0; @@ -532,56 +512,88 @@ impl Host { warn!("Aborted connect. Node already connecting."); return; } - let node = self.nodes.get_mut(id).unwrap(); - node.last_attempted = Some(::time::now()); - - - //blog(NetConnect) << "Attempting connection to node" << _p->id << "@" << ep << "from" << id(); - let socket = match TcpStream::connect(&node.endpoint.address) { - Ok(socket) => socket, - Err(_) => { - warn!("Cannot connect to node"); - return; + + let socket = { + let node = self.nodes.get_mut(id).unwrap(); + node.last_attempted = Some(::time::now()); + + + //blog(NetConnect) << "Attempting connection to node" << _p->id << "@" << ep << "from" << id(); + match TcpStream::connect(&node.endpoint.address) { + Ok(socket) => socket, + Err(_) => { + warn!("Cannot connect to node"); + return; + } } }; - let handshake = Handshake::new(id.clone(), socket); - match self.connecting.insert(handshake) { - Ok(token) => event_loop.register_opt(&self.connecting[token].connection.socket, token, EventSet::all(), PollOpt::edge()).unwrap(), - Err(_) => warn!("Max connections reached") - }; + + let nonce = self.info.next_nonce(); + match self.connecting.insert_with(|token| Handshake::new(token, id, socket, &nonce).expect("Can't create handshake")) { + Some(token) => { + self.connecting[token].register(event_loop).expect("Handshake token regisration failed"); + self.connecting[token].start(&self.info, true); + }, + None => { warn!("Max connections reached") } + } } - fn keep_alive(&mut self) { - } - - fn accept(&mut self, _event_loop: &mut EventLoop) { - warn!(target "net", "accept"); + warn!(target: "net", "accept"); } - fn start_handshake(&mut self, token: Token, _event_loop: &mut EventLoop) { - let handshake = match self.handshakes.get(&token) { - Some(h) => h, - None => { - warn!(target "net", "Received event for unknown handshake"); - return; + fn handshake_writable(&mut self, token: Token, event_loop: &mut EventLoop) { + if !{ + let handshake = match self.connecting.get_mut(token) { + Some(h) => h, + None => { + warn!(target: "net", "Received event for unknown handshake"); + return; + } + }; + match handshake.writable(event_loop, &self.info) { + Err(e) => { + debug!(target: "net", "Handshake read error: {:?}", e); + false + }, + Ok(_) => true } - }; - - - - + } { + self.kill_handshake(token, event_loop); + } + } + fn handshake_readable(&mut self, token: Token, event_loop: &mut EventLoop) { + if !{ + let handshake = match self.connecting.get_mut(token) { + Some(h) => h, + None => { + warn!(target: "net", "Received event for unknown handshake"); + return; + } + }; + match handshake.writable(event_loop, &self.info) { + Err(e) => { + debug!(target: "net", "Handshake read error: {:?}", e); + false + }, + Ok(_) => true + } + } { + self.kill_handshake(token, event_loop); + } + } + fn handshake_timeout(&mut self, token: Token, event_loop: &mut EventLoop) { + self.kill_handshake(token, event_loop) + } + fn kill_handshake(&mut self, token: Token, _event_loop: &mut EventLoop) { + self.connecting.remove(token); } - fn read_handshake(&mut self, _event_loop: &mut EventLoop) { - warn!(target "net", "accept"); + fn read_connection(&mut self, _token: Token, _event_loop: &mut EventLoop) { } - fn read_connection(&mut self, _event_loop: &mut EventLoop) { - } - - fn write_connection(&mut self, _event_loop: &mut EventLoop) { + fn write_connection(&mut self, _token: Token, _event_loop: &mut EventLoop) { } } @@ -594,16 +606,16 @@ impl Handler for Host { match token.as_usize() { TCP_ACCEPT => self.accept(event_loop), IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.read_connection(event_loop), - FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.read_handshake(event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.read_connection(token, event_loop), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(token, event_loop), NODETABLE_RECEIVE => {}, _ => panic!("Received unknown readable token"), } } else if events.is_writable() { match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.write_connection(event_loop), - FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.start_handshake(event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.write_connection(token, event_loop), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(token, event_loop), _ => panic!("Received unknown writable token"), } } @@ -612,6 +624,7 @@ impl Handler for Host { fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { match token.as_usize() { IDLE => self.maintain_network(event_loop), + FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_timeout(token, event_loop), NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, _ => panic!("Received unknown timer token"), diff --git a/src/network/mod.rs b/src/network/mod.rs index 917d79464..b8682dd12 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,2 +1,24 @@ extern crate mio; pub mod host; +pub mod connection; +pub mod handshake; + + +#[derive(Debug)] +pub enum Error { + Crypto(::crypto::CryptoError), + Io(::std::io::Error), +} + +impl From<::std::io::Error> for Error { + fn from(err: ::std::io::Error) -> Error { + Error::Io(err) + } +} + +impl From<::crypto::CryptoError> for Error { + fn from(err: ::crypto::CryptoError) -> Error { + Error::Crypto(err) + } +} + diff --git a/src/sha3.rs b/src/sha3.rs index ee328913c..c466f8915 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -5,6 +5,7 @@ use hash::{FixedHash, H256}; pub trait Hashable { fn sha3(&self) -> H256; + fn sha3_into(&self, dest: &mut [u8]); } impl Hashable for T where T: BytesConvertable { @@ -15,6 +16,9 @@ impl Hashable for T where T: BytesConvertable { ret } } + fn sha3_into(&self, dest: &mut [u8]) { + keccak_256(self.bytes(), dest); + } } #[test] From 84b872819ed68e044e25e763365e4d359388b97b Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 17:23:52 +0100 Subject: [PATCH 141/379] rlp lifetimes --- src/rlp.rs | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index e8562828a..b74651799 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -132,7 +132,7 @@ pub enum Prototype { List(usize), } -impl<'a> Rlp<'a> { +impl<'a, 'view> Rlp<'a> where 'a: 'view { /// Create a new instance of `Rlp` pub fn new(bytes: &'a [u8]) -> Rlp<'a> { Rlp { @@ -162,12 +162,11 @@ impl<'a> Rlp<'a> { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// let view = rlp.at(1); - /// let dog = view.data(); + /// let dog = rlp.at(1).data(); /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&self) -> &[u8] { + pub fn data(&'view self) -> &'a [u8] { self.rlp.data() } @@ -223,7 +222,7 @@ impl<'a> Rlp<'a> { /// assert_eq!(dog, "dog".to_string()); /// } /// ``` - pub fn at(&self, index: usize) -> Rlp<'a> { + pub fn at(&'view self, index: usize) -> Rlp<'a> { From::from(self.rlp.at(index).unwrap()) } @@ -325,7 +324,7 @@ impl<'a> Rlp<'a> { } } -impl<'a> UntrustedRlp<'a> { +impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// returns new instance of `UntrustedRlp` pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { UntrustedRlp { @@ -343,12 +342,11 @@ impl<'a> UntrustedRlp<'a> { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = UntrustedRlp::new(&data); - /// let view = rlp.at(1).unwrap(); - /// let dog = view.data(); + /// let dog = rlp.at(1).unwrap().data(); /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&self) -> &[u8] { + pub fn data(&'view self) -> &'a [u8] { self.bytes } @@ -410,7 +408,7 @@ impl<'a> UntrustedRlp<'a> { /// assert_eq!(dog, "dog".to_string()); /// } /// ``` - pub fn at(&self, index: usize) -> Result, DecoderError> { + pub fn at(&'view self, index: usize) -> Result, DecoderError> { if !self.is_list() { return Err(DecoderError::RlpExpectedToBeList); } @@ -1452,4 +1450,31 @@ mod tests { vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; run_decode_tests(tests); } + + #[test] + fn test_view() { + struct View<'a> { + bytes: &'a [u8] + } + + impl <'a, 'view> View<'a> where 'a: 'view { + fn new(bytes: &'a [u8]) -> View<'a> { + View { + bytes: bytes + } + } + + fn offset(&'view self, len: usize) -> View<'a> { + View::new(&self.bytes[len..]) + } + + fn data(&'view self) -> &'a [u8] { + self.bytes + } + } + + let data = vec![0, 1, 2, 3]; + let view = View::new(&data); + let _data_slice = view.offset(1).data(); + } } From 9736b39363920d941b2a0728a0482cbe4550d2cb Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 17:44:55 +0100 Subject: [PATCH 142/379] fixed benchmarks --- benches/rlp.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 50712bc57..1160e311f 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -20,7 +20,7 @@ fn bench_stream_u64_value(b: &mut Bencher) { // u64 let mut stream = RlpStream::new(); stream.append(&0x1023456789abcdefu64); - let _ = stream.out().unwrap(); + let _ = stream.out(); }); } @@ -42,7 +42,7 @@ fn bench_stream_u256_value(b: &mut Bencher) { stream.append(&U256::from_str("8090a0b0c0d0e0f009102030405060770000000000000001000000000\ 00012f0") .unwrap()); - let _ = stream.out().unwrap(); + let _ = stream.out(); }); } @@ -66,7 +66,7 @@ fn bench_stream_nested_empty_lists(b: &mut Bencher) { stream.append_list(0); stream.append_list(1).append_list(0); stream.append_list(2).append_list(0).append_list(1).append_list(0); - let _ = stream.out().unwrap(); + let _ = stream.out(); }); } @@ -76,11 +76,11 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { // [ [], [[]], [ [], [[]] ] ] let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; let rlp = Rlp::new(&data); - let _v0: Vec = Decodable::decode(&rlp.at(0)); - let _v1: Vec> = Decodable::decode(&rlp.at(1)); + let _v0: Vec = Decodable::decode(&rlp.at(0)); + let _v1: Vec> = Decodable::decode(&rlp.at(1)); let nested_rlp = rlp.at(2); - let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); - let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); + let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); + let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); }); } @@ -91,6 +91,6 @@ fn bench_stream_1000_empty_lists(b: &mut Bencher) { for _ in 0..1000 { stream.append_list(0); } - let _ = stream.out().unwrap(); + let _ = stream.out(); }); } From 90196107284717ca99753660a4e7e20fff1e10fb Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 30 Nov 2015 19:14:46 +0100 Subject: [PATCH 143/379] NibbleSlice mid do not share lifetime with a parent view, but with a collection --- src/nibbleslice.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 6ccf695c0..e7e62e193 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -31,7 +31,7 @@ pub struct NibbleSlice<'a> { offset: usize, } -impl<'a> NibbleSlice<'a> { +impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { /// Create a new nibble slice with the given byte-slice. pub fn new(data: &[u8]) -> NibbleSlice { NibbleSlice::new_offset(data, 0) } @@ -60,7 +60,7 @@ impl<'a> NibbleSlice<'a> { } /// Return object which represents a view on to this slice (further) offset by `i` nibbles. - pub fn mid(&self, i: usize) -> Self { NibbleSlice{ data: self.data, offset: self.offset + i} } + pub fn mid(&'view self, i: usize) -> NibbleSlice<'a> { NibbleSlice{ data: self.data, offset: self.offset + i} } /// Do we start with the same nibbles as the whole of `them`? pub fn starts_with(&self, them: &Self) -> bool { self.common_prefix(them) == them.len() } @@ -192,4 +192,4 @@ mod tests { assert!(n >= m.mid(4)); assert!(n <= m.mid(4)); } -} \ No newline at end of file +} From 2b09521b5632478039cae28311c220b48d6570cb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 01:12:06 +0100 Subject: [PATCH 144/379] Main logic for insertion into trie. --- src/nibbleslice.rs | 12 +++ src/rlp.rs | 6 +- src/trie.rs | 252 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 229 insertions(+), 41 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 6ccf695c0..9656306e5 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -87,6 +87,18 @@ impl<'a> NibbleSlice<'a> { } r } + + pub fn encoded_leftmost(&self, n: usize, is_leaf: bool) -> Bytes { + let l = min(self.len(), n); + let mut r = Bytes::with_capacity(l / 2 + 1); + let mut i = l % 2; + r.push(if i == 1 {0x10 + self.at(0)} else {0} + if is_leaf {0x20} else {0}); + while i < l { + r.push(self.at(i) * 16 + self.at(i + 1)); + i += 2; + } + r + } } impl<'a> PartialEq for NibbleSlice<'a> { diff --git a/src/rlp.rs b/src/rlp.rs index e8562828a..aca12281f 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -167,8 +167,8 @@ impl<'a> Rlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&self) -> &[u8] { - self.rlp.data() + pub fn raw(&self) -> &[u8] { + self.rlp.raw() } /// Returns number of rlp items. @@ -348,7 +348,7 @@ impl<'a> UntrustedRlp<'a> { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn data(&self) -> &[u8] { + pub fn raw(&self) -> &[u8] { self.bytes } diff --git a/src/trie.rs b/src/trie.rs index 4aebc688f..9d004c9dd 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -4,6 +4,7 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; +use log::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); @@ -29,13 +30,44 @@ pub struct TrieDB { root: H256, } -struct Diff { - new: Vec<(H256, Bytes)>, - old: Vec, +enum Operation { + New(H256, Bytes), + Delete(H256), } +struct Diff (Vec) + impl Diff { - pub fn new() -> Diff { Diff { new: vec![], old: vec![] }} + fn new() -> Diff { Diff(vec![]) } + + /// Given the RLP that encodes a node, append a reference to that node `out` and leave `diff` + /// such that the reference is valid, once applied. + fn new_node(&mut self, Bytes rlp, out: &mut RlpStream) { + if (rlp.len() >= 32) { + let rlp_sha3 = rlp.sha3(); + out.append(&rlp_sha3); + self.operations.push(Operation::New(rlp_sha3, rlp)); + } + else { + out.append_raw(&rlp); + } + } + + /// Given the RLP that encodes a now-unused node, leave `diff` in such a state that it is noted. + fn delete_node_sha3(&mut self, old_sha3: H256) { + self.operations.push(Operation::Delete(old_sha3)); + } + + fn delete_node(&mut self, old: &Rlp) { + if (old.is_data() && old.size() == 32) { + self.operations.push(Operation::Delete(H256::decode(old))); + } + } + + fn replace_node(&mut self, old: &Rlp, Bytes rlp, out: &mut RlpStream) { + self.delete_node(old); + self.new_node(rlp, &mut out); + } } impl TrieDB { @@ -57,70 +89,214 @@ impl TrieDB { fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. - let todo = { - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - self.merge(root_rlp, key, value) - }; - self.apply(todo.1); - self.set_root_rlp(&todo.0); + let todo: Diff = Diff::new(); + let root_rlp = self.inject(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); + self.apply(todo); + self.set_root_rlp(&root_rlp); } fn apply(&mut self, diff: Diff) { - for d in diff.old.iter() { - self.db.kill(&d); - } - for d in diff.new.into_iter() { - self.db.emplace(d.0, d.1); + for d in diff.operations.into_iter() { + match d { + Operation::Delete(h) => { + trace!("TrieDB::apply --- {:?}", &h); + self.db.kill(&h); + }, + Operation::New(h, d) => { + trace!("TrieDB::apply +++ {:?} -> {:?}", &h, &d); + self.db.emplace(h, d); + } + } } } - /// Determine the RLP of the node, assuming we're inserting `partial_key` into the - /// node at `old`. This will *not* delete the old mode; it will just return the new RLP - /// that includes the new node. + /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from + /// the trie. + fn take_node(&self, rlp: &Rlp, &mut diff) -> Bytes { + if (rlp.is_data()) { + Bytes::decode(rlp) + } + else { + let h = H256::decode(rlp); + let r = self.db.lookup(&h).as_vec(); + diff.delete_node(h); + r + } + } + + fn inject_and_replace(&self, old: &[u8], old_sha3: H256, partial: &NibbleSlice, value: &[u8], diff: &mut Diff, out: &mut RlpStream) { + diff.new_node(self.inject(old, partial, value, diff), &mut out); + diff.delete_node(old, old_sha3); + } + + /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn transmute_to_branch_and_inject(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + let intermediate = match orig_is_leaf { + true => Self::transmute_leaf_to_branch(orig_partial, orig_raw_payload, &mut diff), + false => Self::transmute_extension_to_branch(orig_partial, orig_raw_payload, &mut diff), + }; + self.inject(&intermediate, partial, value, &mut diff) + // TODO: implement without having to make an intermediate representation. + } + + /// Transform an existing extension or leaf node to an invalid single-entry branch. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RLPStream::new_list(17); + assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. + let index = orig_partial.at(0); + // orig is extension - orig_payload is a node itself. + for i in 0..17 { + if index == i { + if orig_partial.len() > 1 { + // still need an extension + diff.new_node(compose_extension(orig_partial.mid(1), orig_raw_payload), &mut s); + } else { + // was an extension of length 1 - just redirect the payload into here. + s.append_raw(orig_payload.raw()); + } + } else { + s.append_null_data(); + } + } + s.out() + } + + fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RLPStream::new_list(17); + let index = orig_partial.is_empty() ? 16 : orig_partial.at(0); + // orig is leaf - orig_payload is data representing the actual value. + for i in 0..17 { + if index == i { + // this is our node. + diff.new_node(compose_raw(orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), &mut s); + } else { + s.append_null_data(); + } + } + s.out() + } + + /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the + /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that + /// once applied the returned node is valid. + fn injected_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + RlpStream s; + let index = partial.is_empty() ? 16 : partial.at(0); + for i in 0..17 { + if index == i && { + // this is our node. + if (orig.at(i).is_empty()) { + // easy - original had empty slot. + diff.new_node(compose_leaf(partial.mid(if i == 16 {0} else {1}), value), &mut s); + } else if (i == 16) { + // leaf entry - just replace. + let new = compose_leaf(partial.mid(if i == 16 {0} else {1}), value); + diff.replace_node(orig.at(i).raw(), new, &mut s), + } else { + // harder - original has something there already + let new = self.inject(orig.at(i).raw(), partial.mid(1), value, &mut diff); + diff.replace_node(orig.at(i).raw(), new, &mut s) + } + } else { + s.append_raw(orig.at(i).raw()); + } + } + s + } + + /// Determine the RLP of the node, assuming we're inserting `partial` into the + /// node currently of data `old`. This will *not* delete any hash of `old` from the database; + /// it will just return the new RLP that includes the new node. /// /// The database will be updated so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. - fn merge(&self, old: &[u8], partial_key: &NibbleSlice, value: &[u8]) -> (Bytes, Diff) { - let o = Rlp::new(old); - match o.prototype() { + /// + /// **This operation will not insert the new node now destroy the original.** + fn inject(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + // already have an extension. either fast_forward, cleve or transmute_to_branch. + let old_rlp = Rlp::new(old); + match old_rlp.prototype() { Prototype::List(17) => { - // already have a branch. route and merge. - unimplemented!(); + // already have a branch. route and inject. + self.injected_into_branch(old_rlp, partial, value, &mut diff) }, Prototype::List(2) => { - let their_key_rlp = o.at(0); - let (them, _) = NibbleSlice::from_encoded(their_key_rlp.data()); - match partial_key.common_prefix(&them) { + let their_key_rlp = old_rlp.at(0); + let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); + + match partial.common_prefix(&them) { + 0 if partial.is_empty() && them.is_empty() => { + // both empty: just replace. + compose_leaf(partial, value) + }, 0 => { - // transmute to branch here + // one of us isn't empty: transmute to branch here + transmute_to_branch_and_inject(is_leaf, them, old_rlp.at(1).raw()) }, cp if cp == them.len() => { - // fast-forward + // fully-shared prefix for this extension: + // skip to the end of this extension and continue the inject there. + let n = self.take_node(old_rlp.at(1).raw()); + let downstream_node = self.inject(&n, partial.mid(cp), value, &mut diff); + let mut s = RlpStream::new_list(2); + s.append_raw(old_rlp.at(0).raw()); + diff.new_node(downstream_node, &mut s); + s.out() }, - _ => { - // cleve into two + branch in the middle + cp => { + // partially-shared prefix for this extension: + // split into two extensions, high and low, pass the + // low through inject with the value before inserting the result + // into high to create the new. + + // TODO: optimise by doing this without creating injected_low. + + // low (farther from root) + let low = Self::compose_raw(them.mid(cp), old_rlp.at(1).raw(), is_leaf); + let injected_low = self.inject(&low, partial.mid(cp), value, &mut diff); + + // high (closer to root) + let mut s = RlpStream::new_list(2); + s.append(them.encoded_leftmost(cp, false)); + diff.new_node(injected_low, &mut s); + s.out() }, } - // already have an extension. either fast_forward, cleve or transmute_to_branch. - unimplemented!(); }, Prototype::Data(0) => { - (Self::compose_extension(partial_key, value, true), Diff::new()) + (Self::compose_leaf(partial, value, true), Diff::new()) }, _ => panic!("Invalid RLP for node."), } } - fn compose_extension(partial_key: &NibbleSlice, value: &[u8], is_leaf: bool) -> Bytes { - println!("compose_extension {:?} {:?} {:?} ({:?})", partial_key, value, is_leaf, partial_key.encoded(is_leaf)); + fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], bool is_leaf) -> Bytes { + println!("compose_raw {:?} {:?} {:?} ({:?})", partial, value, is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); - s.append(&partial_key.encoded(is_leaf)); - s.append(&value.to_vec()); // WTF?!?! - //s.append(value); // <-- should be. + s.append(&partial.encoded(is_leaf)); + s.append_raw(raw_payload); let r = s.out(); println!("output: -> {:?}", &r); r } + + fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { + println!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); + let mut s = RlpStream::new_list(2); + s.append(&partial.encoded(true)); + s.append(value); + let r = s.out(); + println!("output: -> {:?}", &r); + r + } + + fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { + Self::compose_raw(partial, raw_payload, false) + } } impl Trie for TrieDB { From 1934cb3bdd4b02c9f975f5afa39ae9d53c54211f Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 01:35:32 +0100 Subject: [PATCH 145/379] updated tiny_keccak library --- Cargo.toml | 2 +- src/sha3.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 51e9f4470..de4dcd41b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ arrayvec = "0.3" mio = "0.4.4" rand = "0.3.12" time = "0.1.34" -tiny-keccak = "0.3" +tiny-keccak = "1.0" rocksdb = "0.2.1" num = "0.1" lazy_static = "0.1.*" diff --git a/src/sha3.rs b/src/sha3.rs index ee328913c..11d7ca274 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,5 +1,5 @@ use std::mem::uninitialized; -use tiny_keccak::keccak_256; +use tiny_keccak::Keccak; use bytes::BytesConvertable; use hash::{FixedHash, H256}; @@ -10,8 +10,10 @@ pub trait Hashable { impl Hashable for T where T: BytesConvertable { fn sha3(&self) -> H256 { unsafe { + let mut keccak = Keccak::new_keccak256(); + keccak.update(self.bytes()); let mut ret: H256 = uninitialized(); - keccak_256(self.bytes(), ret.mut_bytes()); + keccak.finalize(ret.mut_bytes()); ret } } From 26f29b2fcdbc25d489ced14478688399fdae9342 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 01:44:18 +0100 Subject: [PATCH 146/379] Logging. --- src/lib.rs | 1 + src/trie.rs | 51 +++++++++++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index e2cb3a754..c52a1b2d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ extern crate num; extern crate log; #[macro_use] extern crate lazy_static; +extern crate env_logger; extern crate time; extern crate crypto as rcrypto; diff --git a/src/trie.rs b/src/trie.rs index b7f9bfa46..f5392f2f1 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -85,7 +85,7 @@ impl TrieDB { fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); - println!("set_root_rlp {:?} {:?}", root_data, self.root); + trace!("set_root_rlp {:?} {:?}", root_data, self.root); } fn apply(&mut self, diff: Diff) { @@ -102,15 +102,26 @@ impl TrieDB { } } } -/* + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. - let todo: Diff = Diff::new(); + let mut todo: Diff = Diff::new(); let root_rlp = self.inject(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); } + fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { + trace!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); + let mut s = RlpStream::new_list(2); + s.append(&partial.encoded(true)); + s.append(&value); + let r = s.out(); + trace!("output: -> {:?}", &r); + r + } + +/* fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], bool is_leaf) -> Bytes { println!("compose_raw {:?} {:?} {:?} ({:?})", partial, value, is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); @@ -121,16 +132,6 @@ impl TrieDB { r } - fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { - println!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); - let mut s = RlpStream::new_list(2); - s.append(&partial.encoded(true)); - s.append(value); - let r = s.out(); - println!("output: -> {:?}", &r); - r - } - fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { Self::compose_raw(partial, raw_payload, false) } @@ -232,6 +233,7 @@ impl TrieDB { diff.new_node(self.inject(old, partial, value, diff), &mut out); diff.delete_node(old, old_sha3); } +*/ /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; @@ -246,11 +248,13 @@ impl TrieDB { let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { + unimplemented!(); // already have a branch. route and inject. - self.injected_into_branch(old_rlp, partial, value, diff) +// self.injected_into_branch(old_rlp, partial, value, diff) }, Prototype::List(2) => { - let their_key_rlp = old_rlp.at(0); + unimplemented!(); +/* let their_key_rlp = old_rlp.at(0); let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); match partial.common_prefix(&them) { @@ -290,15 +294,14 @@ impl TrieDB { diff.new_node(injected_low, s); s.out() }, - } + }*/ }, Prototype::Data(0) => { - (Self::compose_leaf(partial, value, true), Diff::new()) + Self::compose_leaf(partial, value) }, _ => panic!("Invalid RLP for node."), } } - */ } impl Trie for TrieDB { @@ -313,8 +316,7 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - unimplemented!(); -// (self as &mut TrieDB).add(&NibbleSlice::new(key), value); + (self as &mut TrieDB).add(&NibbleSlice::new(key), value); } fn remove(&mut self, _key: &[u8]) { @@ -326,6 +328,9 @@ impl Trie for TrieDB { fn playpen() { use overlaydb::*; use triehash::*; + use env_logger; + + env_logger::init().unwrap(); (&[1, 2, 3]).starts_with(&[1, 2]); @@ -333,7 +338,9 @@ fn playpen() { t.init(); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); -/* + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ]));*/ + assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); + + assert!(false); } From cceae8ecc2a465bc8572fa4730db04e268e3c8cc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 01:52:20 +0100 Subject: [PATCH 147/379] less commented out code in trie. --- src/trie.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index f5392f2f1..58354fcce 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -121,9 +121,8 @@ impl TrieDB { r } -/* - fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], bool is_leaf) -> Bytes { - println!("compose_raw {:?} {:?} {:?} ({:?})", partial, value, is_leaf, partial.encoded(is_leaf)); + fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { + println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload, is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(is_leaf)); s.append_raw(raw_payload, 1); @@ -138,18 +137,21 @@ impl TrieDB { /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. - fn take_node(&self, rlp: &Rlp, &mut diff) -> Bytes { - if (rlp.is_data()) { - Bytes::decode(rlp) + fn take_node(&self, rlp: &Rlp, diff: &mut Diff) -> Bytes { + if (rlp.is_list()) { + rlp.raw().to_vec() } - else { + else if (rlp.is_data() && rlp.size() == 32) { let h = H256::decode(rlp); - let r = self.db.lookup(&h).expect("Trie root not found!").as_vec(); - diff.delete_node(h); + let r = self.db.lookup(&h).expect("Trie root not found!").to_vec(); + diff.delete_node_sha3(h); r } + else { + panic!("Empty or invalid node given?"); + } } - +/* /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** @@ -341,6 +343,4 @@ fn playpen() { t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); - - assert!(false); } From 24f9771716f7b17beaaff309b6e3bec4ecfc84ab Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 02:04:52 +0100 Subject: [PATCH 148/379] fixed take_node lifetimes --- src/trie.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 58354fcce..55cbeb6c5 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -137,13 +137,13 @@ impl TrieDB { /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. - fn take_node(&self, rlp: &Rlp, diff: &mut Diff) -> Bytes { + fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { if (rlp.is_list()) { - rlp.raw().to_vec() + rlp.raw() } else if (rlp.is_data() && rlp.size() == 32) { let h = H256::decode(rlp); - let r = self.db.lookup(&h).expect("Trie root not found!").to_vec(); + let r = self.db.lookup(&h).expect("Trie root not found!"); diff.delete_node_sha3(h); r } From d6e5bbae78fab295229044424e58ca11ff18d20f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 02:23:53 +0100 Subject: [PATCH 149/379] Trie compiles. --- src/trie.rs | 156 ++++++++++++++++++++++++---------------------------- 1 file changed, 72 insertions(+), 84 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 58354fcce..9518cec30 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -5,16 +5,11 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; -use log::*; +//use log::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); -/*lazy_static! { - pub static ref NULL_RLP: Bytes = { let mut r = RlpStream::new(); r.append(&""); r.out().unwrap() }; - pub static ref SHA3_NULL_RLP: H256 = { use sha3::Hashable; NULL_RLP.sha3() }; -}*/ - pub trait Trie { fn root(&self) -> &H256; fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } @@ -39,7 +34,7 @@ impl Diff { /// Given the RLP that encodes a node, append a reference to that node `out` and leave `diff` /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { - if (rlp.len() >= 32) { + if rlp.len() >= 32 { let rlp_sha3 = rlp.sha3(); out.append(&rlp_sha3); self.0.push(Operation::New(rlp_sha3, rlp)); @@ -55,7 +50,7 @@ impl Diff { } fn delete_node(&mut self, old: &Rlp) { - if (old.is_data() && old.size() == 32) { + if old.is_data() && old.size() == 32 { self.0.push(Operation::Delete(H256::decode(old))); } } @@ -138,10 +133,10 @@ impl TrieDB { /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. fn take_node(&self, rlp: &Rlp, diff: &mut Diff) -> Bytes { - if (rlp.is_list()) { + if rlp.is_list() { rlp.raw().to_vec() } - else if (rlp.is_data() && rlp.size() == 32) { + else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); let r = self.db.lookup(&h).expect("Trie root not found!").to_vec(); diff.delete_node_sha3(h); @@ -151,7 +146,46 @@ impl TrieDB { panic!("Empty or invalid node given?"); } } -/* + + /// Transform an existing extension or leaf node to an invalid single-entry branch. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RlpStream::new_list(17); + assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. + let index = orig_partial.at(0); + // orig is extension - orig_raw_payload is a node itself. + for i in 0..17 { + if index == i { + if orig_partial.len() > 1 { + // still need an extension + diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s); + } else { + // was an extension of length 1 - just redirect the payload into here. + s.append_raw(orig_raw_payload, 1); + } + } else { + s.append_empty_data(); + } + } + s.out() + } + + fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + let mut s = RlpStream::new_list(17); + let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; + // orig is leaf - orig_raw_payload is data representing the actual value. + for i in 0..17 { + if index == i { + // this is our node. + diff.new_node(Self::compose_raw(&orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), &mut s); + } else { + s.append_empty_data(); + } + } + s.out() + } + /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** @@ -164,78 +198,35 @@ impl TrieDB { // TODO: implement without having to make an intermediate representation. } - /// Transform an existing extension or leaf node to an invalid single-entry branch. - /// - /// **This operation will not insert the new node nor destroy the original.** - fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { - let mut s = RLPStream::new_list(17); - assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. - let index = orig_partial.at(0); - // orig is extension - orig_payload is a node itself. - for i in 0..17 { - if index == i { - if orig_partial.len() > 1 { - // still need an extension - diff.new_node(compose_extension(orig_partial.mid(1), orig_raw_payload), s); - } else { - // was an extension of length 1 - just redirect the payload into here. - s.append_raw(orig_payload.raw(), 1); - } - } else { - s.append_null_data(); - } - } - s.out() - } - - fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { - let mut s = RLPStream::new_list(17); - let index = orig_partial.is_empty() ? 16 : orig_partial.at(0); - // orig is leaf - orig_payload is data representing the actual value. - for i in 0..17 { - if index == i { - // this is our node. - diff.new_node(compose_raw(orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), s); - } else { - s.append_null_data(); - } - } - s.out() - } - /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that /// once applied the returned node is valid. fn injected_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - RlpStream s; - let index = partial.is_empty() ? 16 : partial.at(0); - for i in 0..17 { - if index == i && { - // this is our node. - if (orig.at(i).is_empty()) { + let mut s = RlpStream::new_list(17); + let index = if partial.is_empty() {16} else {partial.at(0) as usize}; + for i in 0usize..17 { + if index == i { + // this is node to inject into... + if orig.at(i).is_empty() { // easy - original had empty slot. - diff.new_node(compose_leaf(partial.mid(if i == 16 {0} else {1}), value), s); - } else if (i == 16) { + diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s); + } else if i == 16 { // leaf entry - just replace. - let new = compose_leaf(partial.mid(if i == 16 {0} else {1}), value); - diff.replace_node(orig.at(i).raw(), new, s), + let new = Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value); + diff.replace_node(&orig.at(i), new, &mut s); } else { // harder - original has something there already - let new = self.inject(orig.at(i).raw(), partial.mid(1), value, diff); - diff.replace_node(orig.at(i).raw(), new, s) + let new = self.inject(orig.at(i).raw(), &partial.mid(1), value, diff); + diff.replace_node(&orig.at(i), new, &mut s); } } else { s.append_raw(orig.at(i).raw(), 1); } } - s + s.out() } - fn inject_and_replace(&self, old: &[u8], old_sha3: H256, partial: &NibbleSlice, value: &[u8], diff: &mut Diff, out: &mut RlpStream) { - diff.new_node(self.inject(old, partial, value, diff), &mut out); - diff.delete_node(old, old_sha3); - } -*/ + /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; @@ -250,32 +241,29 @@ impl TrieDB { let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { - unimplemented!(); // already have a branch. route and inject. -// self.injected_into_branch(old_rlp, partial, value, diff) + self.injected_into_branch(&old_rlp, partial, value, diff) }, Prototype::List(2) => { - unimplemented!(); -/* let their_key_rlp = old_rlp.at(0); + let their_key_rlp = old_rlp.at(0); let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); - match partial.common_prefix(&them) { 0 if partial.is_empty() && them.is_empty() => { // both empty: just replace. - compose_leaf(partial, value) + Self::compose_leaf(partial, value) }, 0 => { // one of us isn't empty: transmute to branch here - transmute_to_branch_and_inject(is_leaf, them, old_rlp.at(1).raw()) + self.transmute_to_branch_and_inject(is_leaf, &them, old_rlp.at(1).raw(), partial, value, diff) }, cp if cp == them.len() => { // fully-shared prefix for this extension: // skip to the end of this extension and continue the inject there. - let n = self.take_node(old_rlp.at(1).raw()); - let downstream_node = self.inject(&n, partial.mid(cp), value, diff); + let n = self.take_node(&old_rlp.at(1), diff); + let downstream_node = self.inject(&n, &partial.mid(cp), value, diff); let mut s = RlpStream::new_list(2); s.append_raw(old_rlp.at(0).raw(), 1); - diff.new_node(downstream_node, s); + diff.new_node(downstream_node, &mut s); s.out() }, cp => { @@ -287,16 +275,16 @@ impl TrieDB { // TODO: optimise by doing this without creating injected_low. // low (farther from root) - let low = Self::compose_raw(them.mid(cp), old_rlp.at(1).raw(), is_leaf); - let injected_low = self.inject(&low, partial.mid(cp), value, diff); + let low = Self::compose_raw(&them.mid(cp), old_rlp.at(1).raw(), is_leaf); + let injected_low = self.inject(&low, &partial.mid(cp), value, diff); // high (closer to root) let mut s = RlpStream::new_list(2); - s.append(them.encoded_leftmost(cp, false)); - diff.new_node(injected_low, s); + s.append(&them.encoded_leftmost(cp, false)); + diff.new_node(injected_low, &mut s); s.out() }, - }*/ + } }, Prototype::Data(0) => { Self::compose_leaf(partial, value) @@ -318,7 +306,7 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - (self as &mut TrieDB).add(&NibbleSlice::new(key), value); + self.add(&NibbleSlice::new(key), value); } fn remove(&mut self, _key: &[u8]) { From 4e4b754eca3cb782af291032e1b972c9e291ad6d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 03:35:55 +0100 Subject: [PATCH 150/379] First few tests working. --- src/rlp.rs | 9 ++-- src/trie.rs | 117 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 80 insertions(+), 46 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 4d792a15b..c023b1dfb 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -174,7 +174,7 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view { self.rlp.data() } - /// Returns number of rlp items. + /// Returns number of RLP items. /// /// ```rust /// extern crate ethcore_util as util; @@ -210,7 +210,7 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view { self.rlp.size() } - /// Get view onto rlp-slice at index. + /// Get view onto RLP-slice at index. /// /// Caches offset to given index, so access to successive /// slices is faster. @@ -337,7 +337,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } } - /// The bare data of the rlp. + /// The bare data of the RLP. /// /// ```rust /// extern crate ethcore_util as util; @@ -355,7 +355,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } pub fn data(&'view self) -> &'a [u8] { - unimplemented!(); + let ii = Self::item_info(self.bytes).unwrap(); + &self.bytes[ii.prefix_len..(ii.prefix_len + ii.value_len)] } /// Returns number of rlp items. diff --git a/src/trie.rs b/src/trie.rs index 0cf030c75..9f0b209be 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -101,7 +101,7 @@ impl TrieDB { fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); - let root_rlp = self.inject(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); + let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); } @@ -134,11 +134,13 @@ impl TrieDB { /// the trie. fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { + trace!("take_node {:?} (inline)", rlp.raw()); rlp.raw() } else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); let r = self.db.lookup(&h).expect("Trie root not found!"); + trace!("take_node {:?} (indirect for {:?})", rlp.raw(), r); diff.delete_node_sha3(h); r } @@ -150,7 +152,8 @@ impl TrieDB { /// Transform an existing extension or leaf node to an invalid single-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** - fn transmute_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + trace!("transmuted_extension_to_branch"); let mut s = RlpStream::new_list(17); assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. let index = orig_partial.at(0); @@ -171,7 +174,8 @@ impl TrieDB { s.out() } - fn transmute_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + trace!("transmuted_leaf_to_branch"); let mut s = RlpStream::new_list(17); let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; // orig is leaf - orig_raw_payload is data representing the actual value. @@ -189,24 +193,26 @@ impl TrieDB { /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. /// /// **This operation will not insert the new node nor destroy the original.** - fn transmute_to_branch_and_inject(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_to_branch_and_augmented(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + trace!("transmuted_to_branch_and_augmented"); let intermediate = match orig_is_leaf { - true => Self::transmute_leaf_to_branch(orig_partial, orig_raw_payload, diff), - false => Self::transmute_extension_to_branch(orig_partial, orig_raw_payload, diff), + true => Self::transmuted_leaf_to_branch(orig_partial, orig_raw_payload, diff), + false => Self::transmuted_extension_to_branch(orig_partial, orig_raw_payload, diff), }; - self.inject(&intermediate, partial, value, diff) + self.augmented(&intermediate, partial, value, diff) // TODO: implement without having to make an intermediate representation. } /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that /// once applied the returned node is valid. - fn injected_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn augmented_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + trace!("augmented_into_branch"); let mut s = RlpStream::new_list(17); let index = if partial.is_empty() {16} else {partial.at(0) as usize}; for i in 0usize..17 { if index == i { - // this is node to inject into... + // this is node to augment into... if orig.at(i).is_empty() { // easy - original had empty slot. diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s); @@ -216,7 +222,7 @@ impl TrieDB { diff.replace_node(&orig.at(i), new, &mut s); } else { // harder - original has something there already - let new = self.inject(orig.at(i).raw(), &partial.mid(1), value, diff); + let new = self.augmented(orig.at(i).raw(), &partial.mid(1), value, diff); diff.replace_node(&orig.at(i), new, &mut s); } } else { @@ -226,8 +232,6 @@ impl TrieDB { s.out() } - - /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; /// it will just return the new RLP that includes the new node. @@ -236,31 +240,35 @@ impl TrieDB { /// and deleting nodes as necessary. /// /// **This operation will not insert the new node now destroy the original.** - fn inject(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + trace!("augmented ({:?}, {:?}, {:?})", old, partial, value); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { - // already have a branch. route and inject. - self.injected_into_branch(&old_rlp, partial, value, diff) + // already have a branch. route and augment. + self.augmented_into_branch(&old_rlp, partial, value, diff) }, Prototype::List(2) => { - let their_key_rlp = old_rlp.at(0); - let (them, is_leaf) = NibbleSlice::from_encoded(their_key_rlp.data()); - match partial.common_prefix(&them) { - 0 if partial.is_empty() && them.is_empty() => { - // both empty: just replace. + let existing_key_rlp = old_rlp.at(0); + let (existing_key, is_leaf) = NibbleSlice::from_encoded(existing_key_rlp.data()); + match partial.common_prefix(&existing_key) { + cp if partial.len() == existing_key.len() && cp == existing_key.len() && is_leaf => { + // equivalent-leaf: replace + trace!("equivalent-leaf: REPLACE"); Self::compose_leaf(partial, value) }, 0 => { // one of us isn't empty: transmute to branch here - self.transmute_to_branch_and_inject(is_leaf, &them, old_rlp.at(1).raw(), partial, value, diff) + trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); + self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) }, - cp if cp == them.len() => { + cp if cp == existing_key.len() => { + trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: - // skip to the end of this extension and continue the inject there. + // skip to the end of this extension and continue to augment there. let n = self.take_node(&old_rlp.at(1), diff); - let downstream_node = self.inject(n, &partial.mid(cp), value, diff); + let downstream_node = self.augmented(n, &partial.mid(cp), value, diff); let mut s = RlpStream::new_list(2); s.append_raw(old_rlp.at(0).raw(), 1); diff.new_node(downstream_node, &mut s); @@ -269,19 +277,21 @@ impl TrieDB { cp => { // partially-shared prefix for this extension: // split into two extensions, high and low, pass the - // low through inject with the value before inserting the result + // low through augment with the value before inserting the result // into high to create the new. - // TODO: optimise by doing this without creating injected_low. + // TODO: optimise by doing this without creating augmented_low. + + trace!("partially-shared-prefix (exist={:?}; new={:?}; cp={:?}): AUGMENT-AT-END", existing_key.len(), partial.len(), cp); // low (farther from root) - let low = Self::compose_raw(&them.mid(cp), old_rlp.at(1).raw(), is_leaf); - let injected_low = self.inject(&low, &partial.mid(cp), value, diff); + let low = Self::compose_raw(&existing_key.mid(cp), old_rlp.at(1).raw(), is_leaf); + let augmented_low = self.augmented(&low, &partial.mid(cp), value, diff); // high (closer to root) let mut s = RlpStream::new_list(2); - s.append(&them.encoded_leftmost(cp, false)); - diff.new_node(injected_low, &mut s); + s.append(&existing_key.encoded_leftmost(cp, false)); + diff.new_node(augmented_low, &mut s); s.out() }, } @@ -314,21 +324,44 @@ impl Trie for TrieDB { } } -#[test] -fn playpen() { - use overlaydb::*; +#[cfg(test)] +mod tests { + use memorydb::*; use triehash::*; + use super::*; use env_logger; - env_logger::init().unwrap(); + #[test] + fn playpen() { + env_logger::init().unwrap(); - (&[1, 2, 3]).starts_with(&[1, 2]); + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } - let mut t = TrieDB::new(OverlayDB::new_temp()); - t.init(); - assert_eq!(*t.root(), SHA3_NULL_RLP); - assert!(t.is_empty()); + #[test] + fn init() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + assert_eq!(*t.root(), SHA3_NULL_RLP); + assert!(t.is_empty()); + } - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ (vec![1u8, 0x23], vec![1u8, 0x23]) ])); -} + #[test] + fn insert_on_empty() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); + } + + #[test] + fn insert_replace_root() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); + } +} \ No newline at end of file From 0179cf54b7ea271a32a862eb9a4461563ac1f110 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 12:10:36 +0100 Subject: [PATCH 151/379] Better logging, additional tests. --- src/trie.rs | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 9f0b209be..8dbf220ad 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -40,6 +40,7 @@ impl Diff { self.0.push(Operation::New(rlp_sha3, rlp)); } else { + trace!("new_node: inline node {:?}", &rlp); out.append_raw(&rlp, 1); } } @@ -67,13 +68,11 @@ pub struct TrieDB { } impl TrieDB { - pub fn new(db: T) -> Self where T: HashDB + 'static { TrieDB{ db: Box::new(db), root: H256::new() } } + pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r } - pub fn new_boxed(db_box: Box) -> Self { TrieDB{ db: db_box, root: H256::new() } } + pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } - pub fn new_memory() -> Self { TrieDB{ db: Box::new(MemoryDB::new()), root: H256::new() } } - - pub fn init(&mut self) { self.set_root_rlp(&NULL_RLP); } + pub fn new_memory() -> Self { Self::new(MemoryDB::new()) } pub fn db(&self) -> &HashDB { self.db.as_ref() } @@ -84,6 +83,7 @@ impl TrieDB { } fn apply(&mut self, diff: Diff) { + trace!("applying {:?} changes", diff.0.len()); for d in diff.0.into_iter() { match d { Operation::Delete(h) => { @@ -99,11 +99,13 @@ impl TrieDB { } fn add(&mut self, key: &NibbleSlice, value: &[u8]) { + trace!("ADD: {:?} {:?}", key, value); // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); + trace!("---"); } fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { @@ -112,7 +114,7 @@ impl TrieDB { s.append(&partial.encoded(true)); s.append(&value); let r = s.out(); - trace!("output: -> {:?}", &r); + trace!("compose_leaf: -> {:?}", &r); r } @@ -122,7 +124,7 @@ impl TrieDB { s.append(&partial.encoded(is_leaf)); s.append_raw(raw_payload, 1); let r = s.out(); - println!("output: -> {:?}", &r); + println!("compose_raw: -> {:?}", &r); r } @@ -246,6 +248,7 @@ impl TrieDB { let old_rlp = Rlp::new(old); match old_rlp.prototype() { Prototype::List(17) => { + trace!("branch: ROUTE,AUGMENT"); // already have a branch. route and augment. self.augmented_into_branch(&old_rlp, partial, value, diff) }, @@ -297,6 +300,7 @@ impl TrieDB { } }, Prototype::Data(0) => { + trace!("empty: COMPOSE"); Self::compose_leaf(partial, value) }, _ => panic!("Invalid RLP for node."), @@ -326,7 +330,6 @@ impl Trie for TrieDB { #[cfg(test)] mod tests { - use memorydb::*; use triehash::*; use super::*; use env_logger; @@ -335,33 +338,45 @@ mod tests { fn playpen() { env_logger::init().unwrap(); - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + ])); } #[test] fn init() { - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let t = TrieDB::new_memory(); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } #[test] fn insert_on_empty() { - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } #[test] fn insert_replace_root() { - let mut t = TrieDB::new(MemoryDB::new()); - t.init(); + let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); } + + #[test] + fn insert_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + ])); + } } \ No newline at end of file From a90ef806a3a53a5d38610f9da1729e9a9223494e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 12:24:58 +0100 Subject: [PATCH 152/379] Broken triehash --- src/trie.rs | 15 +++++++++++++-- src/triehash.rs | 13 +++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 8dbf220ad..aad6db1cf 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -340,10 +340,21 @@ mod tests { let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + trace!("trieroot with 2 items: {:?}", trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ])); + trace!("trieroot with 3 items: {:?}", trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + ])); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), ])); } diff --git a/src/triehash.rs b/src/triehash.rs index e26cc255f..56df87c55 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -319,6 +319,19 @@ mod tests { assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); } + #[test] + fn broken() { + assert!(trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ]) != + trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + ])); + } + #[test] fn test_trie_root() { let v = vec![ From 6c21bc7b952b10f2935da02a75905a08aa6f8b32 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 12:26:20 +0100 Subject: [PATCH 153/379] out of order test for trieroot --- src/triehash.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 56df87c55..8be23e210 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -320,11 +320,12 @@ mod tests { } #[test] - fn broken() { + fn out_of_order() { assert!(trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - ]) != + ]) == trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), From 9041f81cba0d457ce04f2fd8af594df5f2f33974 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 12:57:08 +0100 Subject: [PATCH 154/379] Additional trie test and fix. --- src/trie.rs | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index aad6db1cf..5c716c947 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -213,22 +213,16 @@ impl TrieDB { let mut s = RlpStream::new_list(17); let index = if partial.is_empty() {16} else {partial.at(0) as usize}; for i in 0usize..17 { - if index == i { - // this is node to augment into... - if orig.at(i).is_empty() { - // easy - original had empty slot. - diff.new_node(Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value), &mut s); - } else if i == 16 { - // leaf entry - just replace. - let new = Self::compose_leaf(&partial.mid(if i == 16 {0} else {1}), value); - diff.replace_node(&orig.at(i), new, &mut s); - } else { - // harder - original has something there already + match (index == i, i) { + (true, 16) => // leaf entry - just replace. + { s.append(&value); }, + (true, i) if orig.at(i).is_empty() => // easy - original had empty slot. + diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), + (true, i) => { // harder - original has something there already let new = self.augmented(orig.at(i).raw(), &partial.mid(1), value, diff); diff.replace_node(&orig.at(i), new, &mut s); } - } else { - s.append_raw(orig.at(i).raw(), 1); + (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, } } s.out() @@ -340,21 +334,10 @@ mod tests { let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); - t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); - trace!("trieroot with 2 items: {:?}", trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - ])); - trace!("trieroot with 3 items: {:?}", trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), - ])); + t.insert(&[], &[0x0]); assert_eq!(*t.root(), trie_root(vec![ + (vec![], vec![0x0]), (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), ])); } @@ -381,7 +364,7 @@ mod tests { } #[test] - fn insert_branch_root() { + fn insert_make_branch_root() { let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); @@ -390,4 +373,17 @@ mod tests { (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) ])); } + + #[test] + fn insert_into_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ])); + } } \ No newline at end of file From 83895ca109bc9e49cc478dcbe6da5773464dfef7 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 16:39:44 +0100 Subject: [PATCH 155/379] trie.at() impl --- src/memorydb.rs | 2 +- src/trie.rs | 169 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 166 insertions(+), 5 deletions(-) diff --git a/src/memorydb.rs b/src/memorydb.rs index 14fae12d0..85fdc7c19 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -189,4 +189,4 @@ fn memorydb_denote() { } assert_eq!(m.lookup(&hash).unwrap(), b"Hello world!"); -} \ No newline at end of file +} diff --git a/src/trie.rs b/src/trie.rs index 9f0b209be..30a186e80 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -16,11 +16,86 @@ pub trait Trie { // TODO: consider returning &[u8]... fn contains(&self, key: &[u8]) -> bool; - fn at(&self, key: &[u8]) -> Option<&[u8]>; + fn at<'a>(&'a self, key: &'a [u8]) -> Option<&'a [u8]>; fn insert(&mut self, key: &[u8], value: &[u8]); fn remove(&mut self, key: &[u8]); } +#[derive(Eq, PartialEq, Debug)] +pub enum Node<'a> { + Leaf(NibbleSlice<'a>, &'a[u8]), + ExtensionRaw(NibbleSlice<'a>, &'a[u8]), + ExtensionSha3(NibbleSlice<'a>, &'a[u8]), + Branch(Option<[&'a[u8]; 16]>, &'a [u8]) +} + +impl <'a>Node<'a> { + pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> { + let r = Rlp::new(node_rlp); + match r.prototype() { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { + (slice, true) => Node::Leaf(slice, r.at(1).data()), + (slice, false) => match r.at(1).raw().len() { + // its just raw extension + 0...31 => Node::ExtensionRaw(slice, r.at(1).raw()), + // its SHA3 of raw extension + the length of sha3 + 33 => Node::ExtensionSha3(slice, r.at(1).data()), + _ => { panic!(); } + } + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { nodes[i] = r.at(i).raw(); } + Node::Branch(Some(nodes), r.at(16).data()) + }, + // an empty branch index. + Prototype::Data(0) => Node::Branch(None, r.data()), + // something went wrong. + _ => panic!("Rlp is not valid.") + } + } + + pub fn encoded(&self) -> Bytes { + match *self { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::ExtensionRaw(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::ExtensionSha3(ref slice, ref sha3) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(false)); + stream.append(sha3); + stream.out() + }, + Node::Branch(Some(ref nodes), ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { stream.append_raw(nodes[i], 1); } + stream.append(value); + stream.out() + }, + Node::Branch(_, _) => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } +} + enum Operation { New(H256, Bytes), Delete(H256), @@ -98,6 +173,30 @@ impl TrieDB { } } + fn get<'a>(&'a self, key: &NibbleSlice<'a>) -> Option<&'a [u8]> { + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + self.get_from_node(&root_rlp, key) + } + + fn get_from_node<'a>(&'a self, node: &'a [u8], key: &NibbleSlice<'a>) -> Option<&'a [u8]> { + match Node::decoded(node) { + Node::Leaf(ref slice, ref value) if key == slice => Some(value), + Node::ExtensionRaw(ref slice, ref item) if key.starts_with(slice) => self.get_from_node(item, &key.mid(slice.len())), + Node::ExtensionSha3(ref slice, ref sha3) => { + // lookup for this item + let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("not found!"); + self.get_from_node(rlp, &key.mid(slice.len())) + }, + Node::Branch(Some(ref nodes), ref value) => match key.is_empty() { + true => Some(value), + // we don't really need to do lookup for nodes[key.at(0)] in db? + // if its empty hash return Nonem without lookup + false => self.get_from_node(nodes[key.at(0) as usize], &key.mid(1)) + }, + _ => None + } + } + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); @@ -311,8 +410,8 @@ impl Trie for TrieDB { unimplemented!(); } - fn at(&self, _key: &[u8]) -> Option<&[u8]> { - unimplemented!(); + fn at<'a>(&'a self, key: &'a [u8]) -> Option<&'a [u8]> { + self.get(&NibbleSlice::new(key)) } fn insert(&mut self, key: &[u8], value: &[u8]) { @@ -329,8 +428,70 @@ mod tests { use memorydb::*; use triehash::*; use super::*; + use nibbleslice::*; + use rlp; use env_logger; + #[test] + fn test_node_leaf() { + let k = vec![0x20u8, 0x01, 0x23, 0x45]; + let v: Vec = From::from("cat"); + let (slice, is_leaf) = NibbleSlice::from_encoded(&k); + assert_eq!(is_leaf, true); + let leaf = Node::Leaf(slice, &v); + let rlp = leaf.encoded(); + let leaf2 = Node::decoded(&rlp); + assert_eq!(leaf, leaf2); + } + + #[test] + fn test_node_extension() { + let k = vec![0x00u8, 0x01, 0x23, 0x45]; + // in extension, value must be valid rlp + let v = rlp::encode(&"cat"); + let (slice, is_leaf) = NibbleSlice::from_encoded(&k); + assert_eq!(is_leaf, false); + let ex = Node::ExtensionRaw(slice, &v); + let rlp = ex.encoded(); + let ex2 = Node::decoded(&rlp); + assert_eq!(ex, ex2); + } + + #[test] + fn test_node_empty_branch() { + let branch = Node::Branch(None, &b""[..]); + let rlp = branch.encoded(); + let branch2 = Node::decoded(&rlp); + assert_eq!(branch, branch2); + } + + #[test] + fn test_node_branch() { + let k = rlp::encode(&"cat"); + let mut nodes: [&[u8]; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { nodes[i] = &k; } + let v: Vec = From::from("dog"); + let branch = Node::Branch(Some(nodes), &v); + let rlp = branch.encoded(); + let branch2 = Node::decoded(&rlp); + assert_eq!(branch, branch2); + } + + #[test] + fn test_at_empty() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + assert_eq!(t.at(&[0x5]), None); + } + + #[test] + fn test_at_one() { + let mut t = TrieDB::new(MemoryDB::new()); + t.init(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); + } + #[test] fn playpen() { env_logger::init().unwrap(); @@ -364,4 +525,4 @@ mod tests { t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); } -} \ No newline at end of file +} From 37c394c99af23ad36ad6e564d02c3b2eda833db5 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 17:03:54 +0100 Subject: [PATCH 156/379] failing insertion test --- src/trie.rs | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index 8cf32037e..bd2be783c 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -181,7 +181,9 @@ impl TrieDB { fn get_from_node<'a>(&'a self, node: &'a [u8], key: &NibbleSlice<'a>) -> Option<&'a [u8]> { match Node::decoded(node) { Node::Leaf(ref slice, ref value) if key == slice => Some(value), - Node::ExtensionRaw(ref slice, ref item) if key.starts_with(slice) => self.get_from_node(item, &key.mid(slice.len())), + Node::ExtensionRaw(ref slice, ref item) if key.starts_with(slice) => { + self.get_from_node(item, &key.mid(slice.len())) + }, Node::ExtensionSha3(ref slice, ref sha3) => { // lookup for this item let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("not found!"); @@ -487,6 +489,41 @@ mod tests { assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); } + #[test] + fn test_at_three() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + assert_eq!(t.at(&[0x01, 0x23]).unwrap(), &[0x01u8, 0x23]); + assert_eq!(t.at(&[0xf1, 0x23]).unwrap(), &[0xf1u8, 0x23]); + assert_eq!(t.at(&[0x81, 0x23]).unwrap(), &[0x81u8, 0x23]); + assert_eq!(t.at(&[0x82, 0x23]), None); + } + + #[test] + fn test_at_dog() { + let mut t = TrieDB::new_memory(); + let v: Vec<(Vec, Vec)> = vec![ + (From::from("do"), From::from("verb")), + (From::from("dog"), From::from("puppy")), + (From::from("doge"), From::from("coin")), + (From::from("horse"), From::from("stallion")), + ]; + + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(&key, &val); + } + + //for i in 0..v.len() { + //let key: &[u8]= &v[i].0; + //let val: &[u8] = &v[i].1; + //assert_eq!(t.at(&key).unwrap(), val); + //} + } + #[test] fn playpen() { env_logger::init().unwrap(); From 3f5bd90b2bb81fec2bdb910b3d3bb87a372180d4 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 17:43:24 +0100 Subject: [PATCH 157/379] Easier debugging output for nibbleslice. --- src/nibbleslice.rs | 12 +++++++- src/trie.rs | 72 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 7569046ad..716963bab 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -1,5 +1,6 @@ //! Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. use std::cmp::*; +use std::fmt; use bytes::*; /// Nibble-orientated view onto byte-slice, allowing nibble-precision offsets. @@ -25,7 +26,7 @@ use bytes::*; /// assert_eq!(n2.mid(3).common_prefix(&n1), 3); /// } /// ``` -#[derive(Debug, Copy, Clone, Eq, Ord)] +#[derive(Copy, Clone, Eq, Ord)] pub struct NibbleSlice<'a> { data: &'a [u8], offset: usize, @@ -122,6 +123,15 @@ impl<'a> PartialOrd for NibbleSlice<'a> { } } +impl<'a> fmt::Debug for NibbleSlice<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in 0..self.len() { + try!(write!(f, "{:01x}", i)); + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::NibbleSlice; diff --git a/src/trie.rs b/src/trie.rs index 5c716c947..d4f2d2f70 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -332,12 +332,14 @@ mod tests { fn playpen() { env_logger::init().unwrap(); + let big_value = b"00000000000000000000000000000000"; + let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[], &[0x0]); + t.insert(&[0x01u8, 0x23], big_value); + t.insert(&[0x11u8, 0x23], big_value); assert_eq!(*t.root(), trie_root(vec![ - (vec![], vec![0x0]), - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) ])); } @@ -386,4 +388,66 @@ mod tests { (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), ])); } + + #[test] + fn insert_value_into_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[], &[0x0]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![], vec![0x0]), + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + ])); + } + + #[test] + fn insert_split_leaf() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), + ])); + } + + #[test] + fn insert_split_extenstion() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01, 0x23, 0x45], &[0x01]); + t.insert(&[0x01, 0xf3, 0x45], &[0x02]); + t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01, 0x23, 0x45], vec![0x01]), + (vec![0x01, 0xf3, 0x45], vec![0x02]), + (vec![0x01, 0xf3, 0xf5], vec![0x03]), + ])); + } + + #[test] + fn insert_big_value() { + let big_value0 = b"00000000000000000000000000000000"; + let big_value1 = b"11111111111111111111111111111111"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value0); + t.insert(&[0x11u8, 0x23], big_value1); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value0.to_vec()), + (vec![0x11u8, 0x23], big_value1.to_vec()) + ])); + } + + #[test] + fn insert_duplicate_value() { + let big_value = b"00000000000000000000000000000000"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value); + t.insert(&[0x11u8, 0x23], big_value); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) + ])); + } } \ No newline at end of file From 7f6efa9a8ffe737747703a73c0f795ae87db71c6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 18:25:18 +0100 Subject: [PATCH 158/379] pretty (hex) printing. --- src/bytes.rs | 32 ++++++++++++++++++++++++++++++++ src/trie.rs | 26 +++++++++++++------------- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index e17ccca9a..a01fda3ee 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -40,6 +40,38 @@ use std::error::Error as StdError; use uint::{U128, U256}; use hash::FixedHash; +pub struct PrettySlice<'a> (&'a [u8]); + +impl<'a> fmt::Debug for PrettySlice<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in 0..self.0.len() { + try!(write!(f, "{:02x}", self.0[i])); + } + Ok(()) + } +} + +pub trait ToPretty { + fn pretty(&self) -> PrettySlice; +} + +impl<'a> ToPretty for &'a [u8] { + fn pretty(&self) -> PrettySlice { + PrettySlice(self) + } +} + +impl<'a> ToPretty for &'a Bytes { + fn pretty(&self) -> PrettySlice { + PrettySlice(self.bytes()) + } +} +impl ToPretty for Bytes { + fn pretty(&self) -> PrettySlice { + PrettySlice(self.bytes()) + } +} + /// Vector of bytes pub type Bytes = Vec; diff --git a/src/trie.rs b/src/trie.rs index bd64be127..1645d9750 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -115,7 +115,7 @@ impl Diff { self.0.push(Operation::New(rlp_sha3, rlp)); } else { - trace!("new_node: inline node {:?}", &rlp); + trace!("new_node: inline node {:?}", rlp.pretty()); out.append_raw(&rlp, 1); } } @@ -154,7 +154,7 @@ impl TrieDB { fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); - trace!("set_root_rlp {:?} {:?}", root_data, self.root); + trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } fn apply(&mut self, diff: Diff) { @@ -166,7 +166,7 @@ impl TrieDB { self.db.kill(&h); }, Operation::New(h, d) => { - trace!("TrieDB::apply +++ {:?} -> {:?}", &h, &d); + trace!("TrieDB::apply +++ {:?} -> {:?}", &h, d.pretty()); self.db.emplace(h, d); } } @@ -200,32 +200,32 @@ impl TrieDB { } fn add(&mut self, key: &NibbleSlice, value: &[u8]) { - trace!("ADD: {:?} {:?}", key, value); + trace!("ADD: {:?} {:?}", key, value.pretty()); // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); - trace!("---"); + trace!("/"); } fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { - trace!("compose_leaf {:?} {:?} ({:?})", partial, value, partial.encoded(true)); + trace!("compose_leaf {:?} {:?} ({:?})", partial, value.pretty(), partial.encoded(true).pretty()); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(true)); s.append(&value); let r = s.out(); - trace!("compose_leaf: -> {:?}", &r); + trace!("compose_leaf: -> {:?}", r.pretty()); r } fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { - println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload, is_leaf, partial.encoded(is_leaf)); + println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(is_leaf)); s.append_raw(raw_payload, 1); let r = s.out(); - println!("compose_raw: -> {:?}", &r); + println!("compose_raw: -> {:?}", r.pretty()); r } @@ -237,18 +237,18 @@ impl TrieDB { /// the trie. fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { - trace!("take_node {:?} (inline)", rlp.raw()); + trace!("take_node {:?} (inline)", rlp.raw().pretty()); rlp.raw() } else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); let r = self.db.lookup(&h).expect("Trie root not found!"); - trace!("take_node {:?} (indirect for {:?})", rlp.raw(), r); + trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); diff.delete_node_sha3(h); r } else { - trace!("take_node {:?} (???)", rlp.raw()); + trace!("take_node {:?} (???)", rlp.raw().pretty()); panic!("Empty or invalid node given?"); } } @@ -339,7 +339,7 @@ impl TrieDB { /// /// **This operation will not insert the new node now destroy the original.** fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("augmented ({:?}, {:?}, {:?})", old, partial, value); + trace!("augmented ({:?}, {:?}, {:?})", old.pretty(), partial, value.pretty()); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); match old_rlp.prototype() { From ea03811a6e76643ac436e464341b4789dd2205e8 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 18:25:47 +0100 Subject: [PATCH 159/379] print trie --- src/trie.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index bd2be783c..efd8134d3 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,3 +1,4 @@ +use std::fmt; use memorydb::*; use sha3::*; use hashdb::*; @@ -142,6 +143,29 @@ pub struct TrieDB { root: H256, } +impl fmt::Debug for TrieDB { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(writeln!(f, "[")); + { + let print_indent = | fo: &mut fmt::Formatter, times | { for i in 0..times { write!(fo, " "); }}; + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + + self.apply_to_all(root_rlp, &mut | node, deepness | { + print_indent(f, deepness); + match *node { + Node::Leaf(ref slice, ref value) => { writeln!(f, "Leaf: {:?}, {:?}", slice, value); }, + Node::ExtensionRaw(ref slice, ref item) => { writeln!(f, "Extension (raw): "); } + Node::ExtensionSha3(ref slice, ref sha3) => { writeln!(f, "Extension (sha3): "); } + Node::Branch(Some(ref nodes), ref value) => { writeln!(f, "Branch: "); } + _ => { writeln!(f, "node"); } + } + }, 0); + } + + writeln!(f, "]") + } +} + impl TrieDB { pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r } @@ -173,6 +197,32 @@ impl TrieDB { } } + fn apply_to_all(&self, node: &[u8], f: &mut F, deepness: usize) where F: FnMut(&Node, usize) -> () { + let node = Node::decoded(node); + match node { + Node::Leaf(_, _) => f(&node, deepness), + Node::ExtensionRaw(_, ref item) => { + f(&node, deepness); + self.apply_to_all(item, f, deepness + 1); + }, + Node::ExtensionSha3(_, sha3) => { + f(&node, deepness); + let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); + self.apply_to_all(rlp, f, deepness + 1); + }, + Node::Branch(Some(ref nodes), ref value) => { + f(&node, deepness); + for i in 0..16 { + self.apply_to_all(nodes[i], f, deepness + 1); + } + }, + // empty + n @ Node::Branch(_, _) => { + // do nothing + } + } + } + fn get<'a>(&'a self, key: &NibbleSlice<'a>) -> Option<&'a [u8]> { let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); self.get_from_node(&root_rlp, key) @@ -186,7 +236,7 @@ impl TrieDB { }, Node::ExtensionSha3(ref slice, ref sha3) => { // lookup for this item - let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("not found!"); + let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); self.get_from_node(rlp, &key.mid(slice.len())) }, Node::Branch(Some(ref nodes), ref value) => match key.is_empty() { @@ -501,6 +551,17 @@ mod tests { assert_eq!(t.at(&[0x82, 0x23]), None); } + #[test] + fn test_print_trie() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + println!("trie:"); + println!("{:?}", t); + assert!(false); + } + #[test] fn test_at_dog() { let mut t = TrieDB::new_memory(); From 4f5399c558c7bac8d5183da372ceef61443f12ef Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 18:43:44 +0100 Subject: [PATCH 160/379] improved printing --- src/trie.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index ebfc4749f..ef87359c3 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -150,16 +150,16 @@ impl fmt::Debug for TrieDB { let print_indent = | fo: &mut fmt::Formatter, times | { for i in 0..times { write!(fo, " "); }}; let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - self.apply_to_all(root_rlp, &mut | node, deepness | { + self.apply_to_all(root_rlp, &mut | node, deepness, index | { print_indent(f, deepness); match *node { - Node::Leaf(ref slice, ref value) => { writeln!(f, "Leaf: {:?}, {:?}", slice, value); }, - Node::ExtensionRaw(ref slice, ref item) => { writeln!(f, "Extension (raw): "); } - Node::ExtensionSha3(ref slice, ref sha3) => { writeln!(f, "Extension (sha3): "); } - Node::Branch(Some(ref nodes), ref value) => { writeln!(f, "Branch: "); } + Node::Leaf(ref slice, ref value) => { writeln!(f, "{:x}: Leaf {:?}, {:?}", index, slice, value); }, + Node::ExtensionRaw(ref slice, ref item) => { write!(f, "Extension (raw): "); } + Node::ExtensionSha3(ref slice, ref sha3) => { write!(f, "Extension (sha3): "); } + Node::Branch(Some(ref nodes), ref value) => { writeln!(f, "{:x} Branch: ", index); } _ => { writeln!(f, "node"); } } - }, 0); + }, 0, 0); } writeln!(f, "]") @@ -197,23 +197,23 @@ impl TrieDB { } } - fn apply_to_all(&self, node: &[u8], f: &mut F, deepness: usize) where F: FnMut(&Node, usize) -> () { + fn apply_to_all(&self, node: &[u8], f: &mut F, deepness: usize, index: usize) where F: FnMut(&Node, usize, usize) -> () { let node = Node::decoded(node); match node { - Node::Leaf(_, _) => f(&node, deepness), + Node::Leaf(_, _) => f(&node, deepness, index), Node::ExtensionRaw(_, ref item) => { - f(&node, deepness); - self.apply_to_all(item, f, deepness + 1); + f(&node, deepness, index); + self.apply_to_all(item, f, deepness + 1, 0); }, Node::ExtensionSha3(_, sha3) => { - f(&node, deepness); + f(&node, deepness, index); let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); - self.apply_to_all(rlp, f, deepness + 1); + self.apply_to_all(rlp, f, deepness + 1, 0); }, Node::Branch(Some(ref nodes), ref value) => { - f(&node, deepness); + f(&node, deepness, index); for i in 0..16 { - self.apply_to_all(nodes[i], f, deepness + 1); + self.apply_to_all(nodes[i], f, deepness + 1, i); } }, // empty @@ -556,6 +556,7 @@ mod tests { fn test_print_trie() { let mut t = TrieDB::new_memory(); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); println!("trie:"); From d2d2f2a8fce69512bf31ffbb1db8cfa182e7e56f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 19:09:48 +0100 Subject: [PATCH 161/379] Fix one bug. --- src/bytes.rs | 5 ++++- src/trie.rs | 32 +++++++++++++++++--------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index a01fda3ee..e81cd8572 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -45,7 +45,10 @@ pub struct PrettySlice<'a> (&'a [u8]); impl<'a> fmt::Debug for PrettySlice<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for i in 0..self.0.len() { - try!(write!(f, "{:02x}", self.0[i])); + match i > 0 { + true => { try!(write!(f, "·{:02x}", self.0[i])); }, + false => { try!(write!(f, "{:02x}", self.0[i])); }, + } } Ok(()) } diff --git a/src/trie.rs b/src/trie.rs index ebfc4749f..41715fb41 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -389,7 +389,7 @@ impl TrieDB { /// /// **This operation will not insert the new node now destroy the original.** fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("augmented ({:?}, {:?}, {:?})", old.pretty(), partial, value.pretty()); + trace!("augmented (old: {:?}, partial: {:?}, value: {:?})", old.pretty(), partial, value.pretty()); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); match old_rlp.prototype() { @@ -401,29 +401,29 @@ impl TrieDB { Prototype::List(2) => { let existing_key_rlp = old_rlp.at(0); let (existing_key, is_leaf) = NibbleSlice::from_encoded(existing_key_rlp.data()); - match partial.common_prefix(&existing_key) { - cp if partial.len() == existing_key.len() && cp == existing_key.len() && is_leaf => { + match (is_leaf, partial.common_prefix(&existing_key)) { + (true, cp) if cp == existing_key.len() && partial.len() == existing_key.len() => { // equivalent-leaf: replace trace!("equivalent-leaf: REPLACE"); Self::compose_leaf(partial, value) }, - 0 => { + (_, 0) => { // one of us isn't empty: transmute to branch here trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) }, - cp if cp == existing_key.len() => { + (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: // skip to the end of this extension and continue to augment there. - let n = self.take_node(&old_rlp.at(1), diff); + let n = if is_leaf { old_rlp.at(1).raw() } else { self.take_node(&old_rlp.at(1), diff) }; let downstream_node = self.augmented(n, &partial.mid(cp), value, diff); let mut s = RlpStream::new_list(2); s.append_raw(old_rlp.at(0).raw(), 1); diff.new_node(downstream_node, &mut s); s.out() }, - cp => { + (_, cp) => { // partially-shared prefix for this extension: // split into two extensions, high and low, pass the // low through augment with the value before inserting the result @@ -445,7 +445,7 @@ impl TrieDB { }, } }, - Prototype::Data(0) => { + Prototype::Data(_) => { trace!("empty: COMPOSE"); Self::compose_leaf(partial, value) }, @@ -570,8 +570,8 @@ mod tests { let v: Vec<(Vec, Vec)> = vec![ (From::from("do"), From::from("verb")), (From::from("dog"), From::from("puppy")), - (From::from("doge"), From::from("coin")), - (From::from("horse"), From::from("stallion")), +// (From::from("doge"), From::from("coin")), +// (From::from("horse"), From::from("stallion")), ]; for i in 0..v.len() { @@ -580,11 +580,13 @@ mod tests { t.insert(&key, &val); } - //for i in 0..v.len() { - //let key: &[u8]= &v[i].0; - //let val: &[u8] = &v[i].1; - //assert_eq!(t.at(&key).unwrap(), val); - //} + assert_eq!(*t.root(), trie_root(v)); + + /*for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + assert_eq!(t.at(&key).unwrap(), val); + }*/ } #[test] From b8b8e4c61ac008f700eb286680f847df006e93ea Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 19:19:16 +0100 Subject: [PATCH 162/379] printing trie --- src/trie.rs | 53 ++++++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index ef87359c3..2680fce5a 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -146,22 +146,8 @@ pub struct TrieDB { impl fmt::Debug for TrieDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "[")); - { - let print_indent = | fo: &mut fmt::Formatter, times | { for i in 0..times { write!(fo, " "); }}; - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - - self.apply_to_all(root_rlp, &mut | node, deepness, index | { - print_indent(f, deepness); - match *node { - Node::Leaf(ref slice, ref value) => { writeln!(f, "{:x}: Leaf {:?}, {:?}", index, slice, value); }, - Node::ExtensionRaw(ref slice, ref item) => { write!(f, "Extension (raw): "); } - Node::ExtensionSha3(ref slice, ref sha3) => { write!(f, "Extension (sha3): "); } - Node::Branch(Some(ref nodes), ref value) => { writeln!(f, "{:x} Branch: ", index); } - _ => { writeln!(f, "node"); } - } - }, 0, 0); - } - + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + self.fmt_all(root_rlp, f, 0); writeln!(f, "]") } } @@ -197,30 +183,47 @@ impl TrieDB { } } - fn apply_to_all(&self, node: &[u8], f: &mut F, deepness: usize, index: usize) where F: FnMut(&Node, usize, usize) -> () { + fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { + for i in 0..size { + try!(write!(f, " ")); + } + Ok(()) + } + + fn fmt_all(&self, node: &[u8], f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { let node = Node::decoded(node); match node { - Node::Leaf(_, _) => f(&node, deepness, index), + Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), Node::ExtensionRaw(_, ref item) => { - f(&node, deepness, index); - self.apply_to_all(item, f, deepness + 1, 0); + self.fmt_indent(f, deepness); + try!(write!(f, "Extension (raw): ")); + self.fmt_all(item, f, deepness + 1); }, Node::ExtensionSha3(_, sha3) => { - f(&node, deepness, index); + self.fmt_indent(f, deepness); + try!(write!(f, "Extension (sha3): ")); let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); - self.apply_to_all(rlp, f, deepness + 1, 0); + self.fmt_all(rlp, f, deepness + 1); }, Node::Branch(Some(ref nodes), ref value) => { - f(&node, deepness, index); + try!(writeln!(f, "Branch: ")); for i in 0..16 { - self.apply_to_all(nodes[i], f, deepness + 1, i); + match Node::decoded(nodes[i]) { + Node::Branch(None, _) => (), + _ => { + self.fmt_indent(f, deepness + 1); + try!(write!(f, "{:x}: ", i)); + self.fmt_all(nodes[i], f, deepness + 1); + } + } } }, // empty n @ Node::Branch(_, _) => { // do nothing } - } + }; + Ok(()) } fn get<'a>(&'a self, key: &NibbleSlice<'a>) -> Option<&'a [u8]> { From 665dfd622e11326ad8bb9d8334a300b1b25d8437 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 19:20:48 +0100 Subject: [PATCH 163/379] // --- src/trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index e696f931a..ca4aeb72a 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -564,7 +564,7 @@ mod tests { t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); println!("trie:"); println!("{:?}", t); - assert!(false); + //assert!(false); } #[test] From 4a71c1266a7511658c3ca89470b81cf811aa0b7d Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 19:24:14 +0100 Subject: [PATCH 164/379] removed build warnings --- src/trie.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index ca4aeb72a..7761aad35 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -147,7 +147,7 @@ impl fmt::Debug for TrieDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "[")); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - self.fmt_all(root_rlp, f, 0); + try!(self.fmt_all(root_rlp, f, 0)); writeln!(f, "]") } } @@ -184,7 +184,7 @@ impl TrieDB { } fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { - for i in 0..size { + for _ in 0..size { try!(write!(f, " ")); } Ok(()) @@ -195,31 +195,31 @@ impl TrieDB { match node { Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), Node::ExtensionRaw(_, ref item) => { - self.fmt_indent(f, deepness); + try!(self.fmt_indent(f, deepness)); try!(write!(f, "Extension (raw): ")); - self.fmt_all(item, f, deepness + 1); + try!(self.fmt_all(item, f, deepness + 1)); }, Node::ExtensionSha3(_, sha3) => { - self.fmt_indent(f, deepness); + try!(self.fmt_indent(f, deepness)); try!(write!(f, "Extension (sha3): ")); let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); - self.fmt_all(rlp, f, deepness + 1); + try!(self.fmt_all(rlp, f, deepness + 1)); }, - Node::Branch(Some(ref nodes), ref value) => { + Node::Branch(Some(ref nodes), _) => { try!(writeln!(f, "Branch: ")); for i in 0..16 { match Node::decoded(nodes[i]) { Node::Branch(None, _) => (), _ => { - self.fmt_indent(f, deepness + 1); + try!(self.fmt_indent(f, deepness + 1)); try!(write!(f, "{:x}: ", i)); - self.fmt_all(nodes[i], f, deepness + 1); + try!(self.fmt_all(nodes[i], f, deepness + 1)); } } } }, // empty - n @ Node::Branch(_, _) => { + Node::Branch(_, _) => { // do nothing } }; @@ -244,8 +244,6 @@ impl TrieDB { }, Node::Branch(Some(ref nodes), ref value) => match key.is_empty() { true => Some(value), - // we don't really need to do lookup for nodes[key.at(0)] in db? - // if its empty hash return Nonem without lookup false => self.get_from_node(nodes[key.at(0) as usize], &key.mid(1)) }, _ => None From 8073adff6867f218b939e15296f41741b54d7af0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 19:45:30 +0100 Subject: [PATCH 165/379] Fix for adding item beyond a leaf. --- src/trie.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 273a11a5a..4b1565329 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -416,8 +416,19 @@ impl TrieDB { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: // skip to the end of this extension and continue to augment there. - let n = if is_leaf { old_rlp.at(1).raw() } else { self.take_node(&old_rlp.at(1), diff) }; - let downstream_node = self.augmented(n, &partial.mid(cp), value, diff); + let downstream_node: Bytes; + if is_leaf { + // TODO: can maybe do with transmuted_to_branch_and_augmented/transmute to branch? + // create mostly-empty branch node. + let mut s = RlpStream::new_list(17); + for _ in 0..16 { s.append_empty_data(); } + s.append_raw(old_rlp.at(1).raw(), 1); + // create rlp for branch with single leaf item. + downstream_node = self.augmented(&s.out(), &partial.mid(cp), value, diff); + } else { + let n = self.take_node(&old_rlp.at(1), diff); + downstream_node = self.augmented(n, &partial.mid(cp), value, diff); + } let mut s = RlpStream::new_list(2); s.append_raw(old_rlp.at(0).raw(), 1); diff.new_node(downstream_node, &mut s); @@ -445,7 +456,7 @@ impl TrieDB { }, } }, - Prototype::Data(_) => { + Prototype::Data(0) => { trace!("empty: COMPOSE"); Self::compose_leaf(partial, value) }, From 6c1c6192479033ed2c6cb144bc4398e6e6a255f2 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 20:24:33 +0100 Subject: [PATCH 166/379] extension print --- src/trie.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 7761aad35..fc4c4835a 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -97,6 +97,11 @@ impl <'a>Node<'a> { } } +//enum ValidationResult<'a> { + //Valid, + //Invalid { node: Node<'a>, depth: usize } +//} + enum Operation { New(H256, Bytes), Delete(H256), @@ -194,14 +199,14 @@ impl TrieDB { let node = Node::decoded(node); match node { Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), - Node::ExtensionRaw(_, ref item) => { + Node::ExtensionRaw(ref slice, ref item) => { try!(self.fmt_indent(f, deepness)); - try!(write!(f, "Extension (raw): ")); + try!(write!(f, "Extension (raw): {:?} ", slice)); try!(self.fmt_all(item, f, deepness + 1)); }, - Node::ExtensionSha3(_, sha3) => { + Node::ExtensionSha3(ref slice, sha3) => { try!(self.fmt_indent(f, deepness)); - try!(write!(f, "Extension (sha3): ")); + try!(write!(f, "Extension (sha3): {:?} ", slice)); let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); try!(self.fmt_all(rlp, f, deepness + 1)); }, From 68065b1875f36894f396482cbbc2a76ae1b4fecb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 20:24:47 +0100 Subject: [PATCH 167/379] Insertion trie fix. --- src/nibbleslice.rs | 5 ++++- src/trie.rs | 43 +++++++++++++++++++++++++------------------ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index bd9ac494d..5f2615991 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -126,7 +126,10 @@ impl<'a> PartialOrd for NibbleSlice<'a> { impl<'a> fmt::Debug for NibbleSlice<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for i in 0..self.len() { - try!(write!(f, "{:01x}", self.at(i))); + match i { + 0 => try!(write!(f, "{:01x}", self.at(i))), + _ => try!(write!(f, "'{:01x}", self.at(i))), + } } Ok(()) } diff --git a/src/trie.rs b/src/trie.rs index d334b85a3..5bae41f49 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -280,10 +280,25 @@ impl TrieDB { r } + fn compose_stub_branch(value: &[u8]) -> Bytes { + let mut s = RlpStream::new_list(17); + for _ in 0..16 { s.append_empty_data(); } + s.append(&value); + s.out() + } + fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { Self::compose_raw(partial, raw_payload, false) } + fn create_extension(partial: &NibbleSlice, downstream_node: Bytes, diff: &mut Diff) -> Bytes { + trace!("create_extension partial: {:?}, downstream_node: {:?}", partial, downstream_node.pretty()); + let mut s = RlpStream::new_list(2); + s.append(&partial.encoded(false)); + diff.new_node(downstream_node, &mut s); + s.out() + } + /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from /// the trie. fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { @@ -416,24 +431,14 @@ impl TrieDB { (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: - // skip to the end of this extension and continue to augment there. - let downstream_node: Bytes; - if is_leaf { - // TODO: can maybe do with transmuted_to_branch_and_augmented/transmute to branch? - // create mostly-empty branch node. - let mut s = RlpStream::new_list(17); - for _ in 0..16 { s.append_empty_data(); } - s.append_raw(old_rlp.at(1).raw(), 1); - // create rlp for branch with single leaf item. - downstream_node = self.augmented(&s.out(), &partial.mid(cp), value, diff); + // transform to an extension + augmented version of onward node. + let downstream_node: Bytes = if is_leaf { + // no onward node because we're a leaf - create fake stub and use that. + self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff) } else { - let n = self.take_node(&old_rlp.at(1), diff); - downstream_node = self.augmented(n, &partial.mid(cp), value, diff); - } - let mut s = RlpStream::new_list(2); - s.append_raw(old_rlp.at(0).raw(), 1); - diff.new_node(downstream_node, &mut s); - s.out() + self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff) + }; + Self::create_extension(&existing_key, downstream_node, diff) }, (_, cp) => { // partially-shared prefix for this extension: @@ -583,7 +588,7 @@ mod tests { let v: Vec<(Vec, Vec)> = vec![ (From::from("do"), From::from("verb")), (From::from("dog"), From::from("puppy")), -// (From::from("doge"), From::from("coin")), + (From::from("doge"), From::from("coin")), // (From::from("horse"), From::from("stallion")), ]; @@ -593,6 +598,8 @@ mod tests { t.insert(&key, &val); } + trace!("{:?}", t); + assert_eq!(*t.root(), trie_root(v)); /*for i in 0..v.len() { From ec09deefb71ed2b4c70168b85aed6fea0270d013 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 20:34:49 +0100 Subject: [PATCH 168/379] printing value of branch --- src/trie.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index fc4c4835a..f3ac095f4 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -190,7 +190,7 @@ impl TrieDB { fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { for _ in 0..size { - try!(write!(f, " ")); + try!(write!(f, " ")); } Ok(()) } @@ -210,8 +210,10 @@ impl TrieDB { let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); try!(self.fmt_all(rlp, f, deepness + 1)); }, - Node::Branch(Some(ref nodes), _) => { + Node::Branch(Some(ref nodes), ref value) => { try!(writeln!(f, "Branch: ")); + try!(self.fmt_indent(f, deepness + 1)); + try!(writeln!(f, ": {:?}", value.pretty())); for i in 0..16 { match Node::decoded(nodes[i]) { Node::Branch(None, _) => (), From 0d90b2c1f8fd2b60fafec687251fd57def389062 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 20:35:34 +0100 Subject: [PATCH 169/379] Minor fixes to trie printing. --- src/trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index ef578ab7a..5e06f2729 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -200,12 +200,12 @@ impl TrieDB { match node { Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), Node::ExtensionRaw(ref slice, ref item) => { - try!(self.fmt_indent(f, deepness)); +// try!(self.fmt_indent(f, deepness)); try!(write!(f, "Extension (raw): {:?} ", slice)); try!(self.fmt_all(item, f, deepness + 1)); }, Node::ExtensionSha3(ref slice, sha3) => { - try!(self.fmt_indent(f, deepness)); +// try!(self.fmt_indent(f, deepness)); try!(write!(f, "Extension (sha3): {:?} ", slice)); let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); try!(self.fmt_all(rlp, f, deepness + 1)); From 67b737c1f8ab87435a9684b686eb912e275bb892 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 22:22:52 +0100 Subject: [PATCH 170/379] Fix one trie bug. --- src/trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 8f160ec7d..c2489240f 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -595,8 +595,8 @@ mod tests { let v: Vec<(Vec, Vec)> = vec![ (From::from("do"), From::from("verb")), (From::from("dog"), From::from("puppy")), - (From::from("doge"), From::from("coin")), -// (From::from("horse"), From::from("stallion")), +// (From::from("doge"), From::from("coin")), + (From::from("horse"), From::from("stallion")), ]; for i in 0..v.len() { From 9a5be1eee4edb1937a9a6c2d8ebf95831773c95b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 22:38:34 +0100 Subject: [PATCH 171/379] Known good. --- src/trie.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index c2489240f..aa57b1b5d 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -116,6 +116,7 @@ impl Diff { /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { if rlp.len() >= 32 { + trace!("new_node: reference node {:?}", rlp.pretty()); let rlp_sha3 = rlp.sha3(); out.append(&rlp_sha3); self.0.push(Operation::New(rlp_sha3, rlp)); @@ -200,12 +201,10 @@ impl TrieDB { match node { Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), Node::ExtensionRaw(ref slice, ref item) => { -// try!(self.fmt_indent(f, deepness)); try!(write!(f, "Extension (raw): {:?} ", slice)); try!(self.fmt_all(item, f, deepness + 1)); }, Node::ExtensionSha3(ref slice, sha3) => { -// try!(self.fmt_indent(f, deepness)); try!(write!(f, "Extension (sha3): {:?} ", slice)); let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); try!(self.fmt_all(rlp, f, deepness + 1)); @@ -595,7 +594,7 @@ mod tests { let v: Vec<(Vec, Vec)> = vec![ (From::from("do"), From::from("verb")), (From::from("dog"), From::from("puppy")), -// (From::from("doge"), From::from("coin")), + (From::from("doge"), From::from("coin")), (From::from("horse"), From::from("stallion")), ]; @@ -605,7 +604,7 @@ mod tests { t.insert(&key, &val); } - trace!("{:?}", t); +// trace!("{:?}", t); assert_eq!(*t.root(), trie_root(v)); From c1c3152094115e50f4d853f8e07a58c7f0ceac0e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 22:44:18 +0100 Subject: [PATCH 172/379] Avoid crying if we can't initialise the logger. --- src/trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index aa57b1b5d..86c2e84cd 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -589,7 +589,7 @@ mod tests { #[test] fn test_at_dog() { - env_logger::init().unwrap(); + env_logger::init().ok(); let mut t = TrieDB::new_memory(); let v: Vec<(Vec, Vec)> = vec![ (From::from("do"), From::from("verb")), @@ -617,7 +617,7 @@ mod tests { #[test] fn playpen() { - env_logger::init().unwrap(); + env_logger::init().ok(); let big_value = b"00000000000000000000000000000000"; From 0cec68b59e99e981c4e72c596fcc6a7ab42be39a Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 23:14:38 +0100 Subject: [PATCH 173/379] fixed trie get and trie print --- src/trie.rs | 75 +++++++++++++++++++++-------------------------------- 1 file changed, 30 insertions(+), 45 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 86c2e84cd..597dbd085 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -25,8 +25,7 @@ pub trait Trie { #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { Leaf(NibbleSlice<'a>, &'a[u8]), - ExtensionRaw(NibbleSlice<'a>, &'a[u8]), - ExtensionSha3(NibbleSlice<'a>, &'a[u8]), + Extension(NibbleSlice<'a>, &'a[u8]), Branch(Option<[&'a[u8]; 16]>, &'a [u8]) } @@ -41,13 +40,7 @@ impl <'a>Node<'a> { // fed back into this function or inline RLP which can be fed back into this function). Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { (slice, true) => Node::Leaf(slice, r.at(1).data()), - (slice, false) => match r.at(1).raw().len() { - // its just raw extension - 0...31 => Node::ExtensionRaw(slice, r.at(1).raw()), - // its SHA3 of raw extension + the length of sha3 - 33 => Node::ExtensionSha3(slice, r.at(1).data()), - _ => { panic!(); } - } + (slice, false) => Node::Extension(slice, r.at(1).raw()) }, // branch - first 16 are nodes, 17th is a value (or empty). Prototype::List(17) => { @@ -62,6 +55,7 @@ impl <'a>Node<'a> { } } + // todo: should check length before encoding, cause it may just be sha3 of data pub fn encoded(&self) -> Bytes { match *self { Node::Leaf(ref slice, ref value) => { @@ -70,18 +64,12 @@ impl <'a>Node<'a> { stream.append(value); stream.out() }, - Node::ExtensionRaw(ref slice, ref raw_rlp) => { + Node::Extension(ref slice, ref raw_rlp) => { let mut stream = RlpStream::new_list(2); stream.append(&slice.encoded(false)); stream.append_raw(raw_rlp, 1); stream.out() }, - Node::ExtensionSha3(ref slice, ref sha3) => { - let mut stream = RlpStream::new_list(2); - stream.append(&slice.encoded(false)); - stream.append(sha3); - stream.out() - }, Node::Branch(Some(ref nodes), ref value) => { let mut stream = RlpStream::new_list(17); for i in 0..16 { stream.append_raw(nodes[i], 1); } @@ -200,27 +188,21 @@ impl TrieDB { let node = Node::decoded(node); match node { Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), - Node::ExtensionRaw(ref slice, ref item) => { - try!(write!(f, "Extension (raw): {:?} ", slice)); - try!(self.fmt_all(item, f, deepness + 1)); - }, - Node::ExtensionSha3(ref slice, sha3) => { - try!(write!(f, "Extension (sha3): {:?} ", slice)); - let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); - try!(self.fmt_all(rlp, f, deepness + 1)); + Node::Extension(ref slice, ref item) => { + try!(write!(f, "Extension: {:?} ,", slice)); + try!(self.fmt_all(self.get_raw_or_lookup(item), f, deepness + 1)); }, Node::Branch(Some(ref nodes), ref value) => { try!(writeln!(f, "Branch: ")); try!(self.fmt_indent(f, deepness + 1)); try!(writeln!(f, ": {:?}", value.pretty())); for i in 0..16 { - match Node::decoded(nodes[i]) { - Node::Branch(None, _) => (), - _ => { - try!(self.fmt_indent(f, deepness + 1)); - try!(write!(f, "{:x}: ", i)); - try!(self.fmt_all(nodes[i], f, deepness + 1)); - } + let rlp = Rlp::new(nodes[i]); + if rlp.is_data() && rlp.size() != 0 { + try!(self.fmt_indent(f, deepness + 1)); + try!(write!(f, "{:x}: ", i)); + try!(self.fmt_all(self.get_raw_or_lookup(nodes[i]), f, deepness + 1)); + try!(writeln!(f, "")); } } }, @@ -240,22 +222,25 @@ impl TrieDB { fn get_from_node<'a>(&'a self, node: &'a [u8], key: &NibbleSlice<'a>) -> Option<&'a [u8]> { match Node::decoded(node) { Node::Leaf(ref slice, ref value) if key == slice => Some(value), - Node::ExtensionRaw(ref slice, ref item) if key.starts_with(slice) => { - self.get_from_node(item, &key.mid(slice.len())) - }, - Node::ExtensionSha3(ref slice, ref sha3) => { - // lookup for this item - let rlp = self.db.lookup(&H256::from_slice(sha3)).expect("sha3 not found!"); - self.get_from_node(rlp, &key.mid(slice.len())) + Node::Extension(ref slice, ref item) if key.starts_with(slice) => { + self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len())) }, Node::Branch(Some(ref nodes), ref value) => match key.is_empty() { true => Some(value), - false => self.get_from_node(nodes[key.at(0) as usize], &key.mid(1)) + false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1)) }, _ => None } } + fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { + // check if its sha3 + len + match node.len() > 32 { + true => self.db.lookup(&H256::from_slice(&node[1..])).expect("Not found!"), + false => node + } + } + fn add(&mut self, key: &NibbleSlice, value: &[u8]) { trace!("ADD: {:?} {:?}", key, value.pretty()); // determine what the new root is, insert new nodes and remove old as necessary. @@ -524,7 +509,7 @@ mod tests { let v = rlp::encode(&"cat"); let (slice, is_leaf) = NibbleSlice::from_encoded(&k); assert_eq!(is_leaf, false); - let ex = Node::ExtensionRaw(slice, &v); + let ex = Node::Extension(slice, &v); let rlp = ex.encoded(); let ex2 = Node::decoded(&rlp); assert_eq!(ex, ex2); @@ -604,15 +589,15 @@ mod tests { t.insert(&key, &val); } -// trace!("{:?}", t); + println!("{:?}", t); - assert_eq!(*t.root(), trie_root(v)); - - /*for i in 0..v.len() { + for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; assert_eq!(t.at(&key).unwrap(), val); - }*/ + } + + assert_eq!(*t.root(), trie_root(v)); } #[test] From 6ba9af18e9dc14742f5be7cd8fb13c9d36182f07 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 1 Dec 2015 23:15:23 +0100 Subject: [PATCH 174/379] Fiddling with tests. --- src/trie.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 86c2e84cd..c0907f7f3 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -393,7 +393,7 @@ impl TrieDB { (true, i) if orig.at(i).is_empty() => // easy - original had empty slot. diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), (true, i) => { // harder - original has something there already - let new = self.augmented(orig.at(i).raw(), &partial.mid(1), value, diff); + let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); diff.replace_node(&orig.at(i), new, &mut s); } (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, @@ -472,7 +472,7 @@ impl TrieDB { trace!("empty: COMPOSE"); Self::compose_leaf(partial, value) }, - _ => panic!("Invalid RLP for node."), + _ => panic!("Invalid RLP for node: {:?}", old.pretty()), } } } @@ -591,11 +591,12 @@ mod tests { fn test_at_dog() { env_logger::init().ok(); let mut t = TrieDB::new_memory(); - let v: Vec<(Vec, Vec)> = vec![ + let mut v: Vec<(Vec, Vec)> = vec![ (From::from("do"), From::from("verb")), (From::from("dog"), From::from("puppy")), (From::from("doge"), From::from("coin")), (From::from("horse"), From::from("stallion")), + (From::from("dot"), From::from("point")), ]; for i in 0..v.len() { @@ -604,8 +605,9 @@ mod tests { t.insert(&key, &val); } -// trace!("{:?}", t); +// trace!("{:?}", t); + v.sort(); assert_eq!(*t.root(), trie_root(v)); /*for i in 0..v.len() { From beaf9117d40ae4aca94f80d65a13c7a484cd9086 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 1 Dec 2015 23:32:23 +0100 Subject: [PATCH 175/379] lifetime of tria.at --- src/trie.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 597dbd085..88cc390e8 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -17,7 +17,7 @@ pub trait Trie { // TODO: consider returning &[u8]... fn contains(&self, key: &[u8]) -> bool; - fn at<'a>(&'a self, key: &'a [u8]) -> Option<&'a [u8]>; + fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; fn insert(&mut self, key: &[u8], value: &[u8]); fn remove(&mut self, key: &[u8]); } @@ -214,12 +214,12 @@ impl TrieDB { Ok(()) } - fn get<'a>(&'a self, key: &NibbleSlice<'a>) -> Option<&'a [u8]> { + fn get<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); self.get_from_node(&root_rlp, key) } - fn get_from_node<'a>(&'a self, node: &'a [u8], key: &NibbleSlice<'a>) -> Option<&'a [u8]> { + fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { match Node::decoded(node) { Node::Leaf(ref slice, ref value) if key == slice => Some(value), Node::Extension(ref slice, ref item) if key.starts_with(slice) => { @@ -469,7 +469,7 @@ impl Trie for TrieDB { unimplemented!(); } - fn at<'a>(&'a self, key: &'a [u8]) -> Option<&'a [u8]> { + fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { self.get(&NibbleSlice::new(key)) } @@ -597,6 +597,9 @@ mod tests { assert_eq!(t.at(&key).unwrap(), val); } + // check lifetime + let _q = t.at(&[b'd', b'o']).unwrap(); + assert_eq!(*t.root(), trie_root(v)); } From 21730b49e6b8a3f3042d354f182b0654d14c06f1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 00:14:14 +0100 Subject: [PATCH 176/379] Fix up trie pretty print. --- src/trie.rs | 92 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index d654fd31f..390448c53 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -24,9 +24,10 @@ pub trait Trie { #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { + NullRoot, Leaf(NibbleSlice<'a>, &'a[u8]), Extension(NibbleSlice<'a>, &'a[u8]), - Branch(Option<[&'a[u8]; 16]>, &'a [u8]) + Branch([Option<&'a[u8]>; 16], Option<&'a [u8]>) } impl <'a>Node<'a> { @@ -40,16 +41,18 @@ impl <'a>Node<'a> { // fed back into this function or inline RLP which can be fed back into this function). Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { (slice, true) => Node::Leaf(slice, r.at(1).data()), - (slice, false) => Node::Extension(slice, r.at(1).raw()) + (slice, false) => Node::Extension(slice, r.at(1).raw()), }, // branch - first 16 are nodes, 17th is a value (or empty). Prototype::List(17) => { - let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; - for i in 0..16 { nodes[i] = r.at(i).raw(); } - Node::Branch(Some(nodes), r.at(16).data()) + let mut nodes: [Option<&'a [u8]>; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { + nodes[i] = if r.at(i).is_empty() { None } else { Some(r.at(i).raw()) } + } + Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) }, // an empty branch index. - Prototype::Data(0) => Node::Branch(None, r.data()), + Prototype::Data(0) => Node::NullRoot, // something went wrong. _ => panic!("Rlp is not valid.") } @@ -70,13 +73,21 @@ impl <'a>Node<'a> { stream.append_raw(raw_rlp, 1); stream.out() }, - Node::Branch(Some(ref nodes), ref value) => { + Node::Branch(ref nodes, ref value) => { let mut stream = RlpStream::new_list(17); - for i in 0..16 { stream.append_raw(nodes[i], 1); } - stream.append(value); + for i in 0..16 { + match nodes[i] { + Some(n) => { stream.append_raw(n, 1); }, + None => { stream.append_empty_data(); }, + } + } + match *value { + Some(n) => { stream.append(&n); }, + None => { stream.append_empty_data(); }, + } stream.out() }, - Node::Branch(_, _) => { + Node::NullRoot => { let mut stream = RlpStream::new(); stream.append_empty_data(); stream.out() @@ -187,29 +198,34 @@ impl TrieDB { fn fmt_all(&self, node: &[u8], f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { let node = Node::decoded(node); match node { - Node::Leaf(slice, value) => try!(writeln!(f, "Leaf {:?}, {:?}", slice, value.pretty())), + Node::Leaf(slice, value) => try!(writeln!(f, "-{:?}: {:?}.", slice, value.pretty())), Node::Extension(ref slice, ref item) => { - try!(write!(f, "Extension: {:?} ,", slice)); - try!(self.fmt_all(self.get_raw_or_lookup(item), f, deepness + 1)); + try!(write!(f, "-{:?}- ", slice)); + try!(self.fmt_all(self.get_raw_or_lookup(item), f, deepness)); }, - Node::Branch(Some(ref nodes), ref value) => { - try!(writeln!(f, "Branch: ")); - try!(self.fmt_indent(f, deepness + 1)); - if (!value.is_empty()) - try!(writeln!(f, "= {:?}", value.pretty())); - for i in 0..16 { - let rlp = Rlp::new(nodes[i]); - if rlp.is_data() && rlp.size() != 0 { + Node::Branch(ref nodes, ref value) => { + try!(writeln!(f, "-< ")); + match value { + &Some(v) => { try!(self.fmt_indent(f, deepness + 1)); - try!(write!(f, "{:x}: ", i)); - try!(self.fmt_all(self.get_raw_or_lookup(nodes[i]), f, deepness + 1)); - try!(writeln!(f, "")); + try!(writeln!(f, "=: {:?}", v.pretty())) + }, + &None => {} + } + for i in 0..16 { + match nodes[i] { + Some(n) => { + try!(self.fmt_indent(f, deepness + 1)); + try!(write!(f, "{:x}: ", i)); + try!(self.fmt_all(self.get_raw_or_lookup(n), f, deepness + 1)); + }, + None => {}, } } }, // empty - Node::Branch(_, _) => { - // do nothing + Node::NullRoot => { + writeln!(f, ""); } }; Ok(()) @@ -226,9 +242,14 @@ impl TrieDB { Node::Extension(ref slice, ref item) if key.starts_with(slice) => { self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len())) }, - Node::Branch(Some(ref nodes), ref value) => match key.is_empty() { - true => Some(value), - false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1)) + Node::Branch(ref nodes, value) => match key.is_empty() { + true => value, + false => match nodes[key.at(0) as usize] { + Some(payload) => { + self.get_from_node(self.get_raw_or_lookup(payload), &key.mid(1)) + }, + None => None + } }, _ => None } @@ -236,8 +257,9 @@ impl TrieDB { fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { // check if its sha3 + len - match node.len() > 32 { - true => self.db.lookup(&H256::from_slice(&node[1..])).expect("Not found!"), + let r = Rlp::new(node); + match r.is_data() && r.size() == 32 { + true => self.db.lookup(&H256::decode(&r)).expect("Not found!"), false => node } } @@ -518,7 +540,7 @@ mod tests { #[test] fn test_node_empty_branch() { - let branch = Node::Branch(None, &b""[..]); + let branch = Node::Branch([None; 16], None); let rlp = branch.encoded(); let branch2 = Node::decoded(&rlp); assert_eq!(branch, branch2); @@ -527,10 +549,10 @@ mod tests { #[test] fn test_node_branch() { let k = rlp::encode(&"cat"); - let mut nodes: [&[u8]; 16] = unsafe { ::std::mem::uninitialized() }; - for i in 0..16 { nodes[i] = &k; } + let mut nodes: [Option<&[u8]>; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { nodes[i] = Some(&k); } let v: Vec = From::from("dog"); - let branch = Node::Branch(Some(nodes), &v); + let branch = Node::Branch(nodes, Some(&v)); let rlp = branch.encoded(); let branch2 = Node::decoded(&rlp); assert_eq!(branch, branch2); From 73be98fb24a006bc1e84aff7a7e406a8d1062f0f Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 00:21:03 +0100 Subject: [PATCH 177/379] additional tests for insert --- src/trie.rs | 63 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 88cc390e8..30f8c152e 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -465,8 +465,8 @@ impl TrieDB { impl Trie for TrieDB { fn root(&self) -> &H256 { &self.root } - fn contains(&self, _key: &[u8]) -> bool { - unimplemented!(); + fn contains(&self, key: &[u8]) -> bool { + self.at(key).is_some() } fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { @@ -484,6 +484,7 @@ impl Trie for TrieDB { #[cfg(test)] mod tests { + use rustc_serialize::hex::FromHex; use triehash::*; use super::*; use nibbleslice::*; @@ -572,17 +573,9 @@ mod tests { //assert!(false); } - #[test] - fn test_at_dog() { - env_logger::init().ok(); + fn test_all(v: Vec<(Vec, Vec)>) { let mut t = TrieDB::new_memory(); - let v: Vec<(Vec, Vec)> = vec![ - (From::from("do"), From::from("verb")), - (From::from("dog"), From::from("puppy")), - (From::from("doge"), From::from("coin")), - (From::from("horse"), From::from("stallion")), - ]; - + for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -603,6 +596,52 @@ mod tests { assert_eq!(*t.root(), trie_root(v)); } + #[test] + fn test_at_dog() { + env_logger::init().ok(); + let v = vec![ + (From::from("do"), From::from("verb")), + (From::from("dog"), From::from("puppy")), + (From::from("doge"), From::from("coin")), + (From::from("horse"), From::from("stallion")), + ]; + + test_all(v); + } + + #[test] + fn test_more_data() { + let v = vec![ + + ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), + + ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), + "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), + "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), + + ("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), + "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), + "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), + + ("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), + + ("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), + "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), + + ("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), + "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) + + ]; + + test_all(v); + } + #[test] fn playpen() { env_logger::init().ok(); From 5468210cfd38c374e9c11ee29facc4e3c2edac64 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 00:43:09 +0100 Subject: [PATCH 178/379] Fix bad test. --- src/trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index d6afb3bc8..4fc2b3a70 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -609,7 +609,7 @@ mod tests { println!("{:?}", t); // check lifetime - let _q = t.at(&[b'd', b'o']).unwrap(); +// let _q = t.at(&[b'd', b'o']).unwrap(); assert_eq!(*t.root(), trie_root(v)); } From fbab7f3cf36ce30791448bb0a7b6cfa943bfebac Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 02:37:26 +0100 Subject: [PATCH 179/379] Random trie tests. --- src/trie.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 4fc2b3a70..2fb05ccfb 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,3 +1,5 @@ +extern crate rand; + use std::fmt; use memorydb::*; use sha3::*; @@ -6,6 +8,7 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; + //use log::*; pub const NULL_RLP: [u8; 1] = [0x80; 1]; @@ -509,10 +512,12 @@ impl Trie for TrieDB { mod tests { use rustc_serialize::hex::FromHex; use triehash::*; + use hash::*; use super::*; use nibbleslice::*; use rlp; use env_logger; + use rand::random; #[test] fn test_node_leaf() { @@ -605,8 +610,8 @@ mod tests { t.insert(&key, &val); } - trace!("{:?}", t); - println!("{:?}", t); +// trace!("{:?}", t); +// println!("{:?}", t); // check lifetime // let _q = t.at(&[b'd', b'o']).unwrap(); @@ -614,6 +619,43 @@ mod tests { assert_eq!(*t.root(), trie_root(v)); } + fn random_key() -> Vec { + let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; + let mut ret: Vec = Vec::new(); + let r = random::() % 4 + 1; + for _ in 0..r { + ret.push(chars[random::() % chars.len()]); + } + ret + } + + #[test] + fn stress() { + for _ in 0..1000 { + let mut x: Vec<(Vec, Vec)> = Vec::new(); + for j in 0..100u32 { + let key = random_key(); + x.push((key, rlp::encode(&j))); + } + let real = trie_root(x.clone()); + + let mem = trie_root_mem(&x); + assert_eq!(mem, real); + } + } + + fn trie_root_mem(v: &Vec<(Vec, Vec)>) -> H256 { + let mut t = TrieDB::new_memory(); + + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(&key, &val); + } + + t.root().clone() + } + #[test] fn test_at_dog() { env_logger::init().ok(); From 706428acbe134d54553bf3fb52bfcd344c707e11 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 02:44:45 +0100 Subject: [PATCH 180/379] Random tests. --- src/triehash.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index afc07a7bf..33d35513c 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -190,9 +190,12 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // iterate over all possible nibbles for i in 0..16 { // cout how many successive elements have same next nibble - let len = input[begin..].iter() - .map(| pair | pair.0[pre_len] ) - .take_while(|&q| q == i).count(); + let len = match begin < input.len() { + true => input[begin..].iter() + .map(| pair | pair.0[pre_len] ) + .take_while(|&q| q == i).count(), + false => 0 + }; // if at least 1 successive element has the same nibble // append their suffixes From 10a43c1fed5ad825b7d9ee520ce7ed2ff8b97ea9 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 2 Dec 2015 12:07:46 +0100 Subject: [PATCH 181/379] Encrypted connection --- src/crypto.rs | 55 +++++++++- src/network/connection.rs | 213 +++++++++++++++++++++++++++++++++++- src/network/handshake.rs | 161 ++++++++++++++++++--------- src/network/host.rs | 224 ++++++++++++++++++++------------------ src/network/mod.rs | 32 +++++- src/network/session.rs | 33 ++++++ src/sha3.rs | 4 +- 7 files changed, 552 insertions(+), 170 deletions(-) create mode 100644 src/network/session.rs diff --git a/src/crypto.rs b/src/crypto.rs index 6843a204b..0ffd0f876 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -48,10 +48,10 @@ impl From<::std::io::Error> for CryptoError { /// fn main() { /// let pair = KeyPair::create().unwrap(); /// let message = H256::random(); -/// let signature = sign(pair.secret(), &message).unwrap(); +/// let signature = ec::sign(pair.secret(), &message).unwrap(); /// -/// assert!(verify(pair.public(), &signature, &message).unwrap()); -/// assert_eq!(recover(&signature, &message).unwrap(), *pair.public()); +/// assert!(ec::verify(pair.public(), &signature, &message).unwrap()); +/// assert_eq!(ec::recover(&signature, &message).unwrap(), *pair.public()); /// } /// ``` pub struct KeyPair { @@ -200,6 +200,49 @@ pub mod ecies { Ok(msg) } + pub fn decrypt(secret: &Secret, encrypted: &[u8]) -> Result { + use ::rcrypto::digest::Digest; + use ::rcrypto::sha2::Sha256; + use ::rcrypto::hmac::Hmac; + use ::rcrypto::mac::Mac; + + let meta_len = encrypted.len() - (1 + 64 + 16 + 32); + if encrypted.len() < meta_len || encrypted[0] < 2 || encrypted[0] > 4 { + return Err(CryptoError::InvalidMessage); //invalid message: publickey + } + + let e = &encrypted[1..]; + let p = Public::from_slice(&e[0..64]); + let z = try!(ecdh::agree(secret, &p)); + let mut key = [0u8; 32]; + kdf(&z, &[0u8; 0], &mut key); + let ekey = &key[0..16]; + let mkey_material = &key[16..32]; + let mut hasher = Sha256::new(); + let mut mkey = [0u8; 32]; + hasher.input(mkey_material); + hasher.result(&mut mkey); + + let clen = encrypted.len() - meta_len; + let cypher_with_iv = &e[64..(64+16+clen)]; + let cypher_iv = &cypher_with_iv[0..16]; + let cypher_no_iv = &cypher_with_iv[16..]; + let msg_mac = &e[(64+16+clen)..]; + + // Verify tag + let mut hmac = Hmac::new(Sha256::new(), &mkey); + hmac.input(cypher_iv); + let mut mac = H256::new(); + hmac.raw_result(&mut mac); + if &mac[..] != msg_mac { + return Err(CryptoError::InvalidMessage); + } + + let mut msg = vec![0u8; clen]; + aes::decrypt(ekey, &H128::new(), cypher_no_iv, &mut msg[..]); + Ok(msg) + } + fn kdf(secret: &Secret, s1: &[u8], dest: &mut [u8]) { use ::rcrypto::digest::Digest; use ::rcrypto::sha2::Sha256; @@ -252,10 +295,10 @@ mod tests { fn test_signature() { let pair = KeyPair::create().unwrap(); let message = H256::random(); - let signature = sign(pair.secret(), &message).unwrap(); + let signature = ec::sign(pair.secret(), &message).unwrap(); - assert!(verify(pair.public(), &signature, &message).unwrap()); - assert_eq!(recover(&signature, &message).unwrap(), *pair.public()); + assert!(ec::verify(pair.public(), &signature, &message).unwrap()); + assert_eq!(ec::recover(&signature, &message).unwrap(), *pair.public()); } #[test] diff --git a/src/network/connection.rs b/src/network/connection.rs index c5deddf58..db5cde10d 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -1,9 +1,22 @@ -use std::io::{self, Cursor, Read}; -use mio::*; +#![allow(dead_code)] //TODO: remove this after everything is done +use mio::{Token, EventSet, EventLoop, Timeout, PollOpt, TryRead, TryWrite}; use mio::tcp::*; use hash::*; +use sha3::*; use bytes::*; -use network::host::Host; +use rlp::*; +use std::io::{self, Cursor, Read}; +use network::host::{Host}; +use network::Error; +use network::handshake::Handshake; +use crypto; +use rcrypto::blockmodes::*; +use rcrypto::aessafe::*; +use rcrypto::symmetriccipher::*; +use rcrypto::buffer::*; +use tiny_keccak::Keccak; + +const ENCRYPTED_HEADER_LEN: usize = 32; pub struct Connection { pub token: Token, @@ -19,6 +32,189 @@ pub enum WriteStatus { Complete } +enum EncryptedConnectionState { + Header, + Payload, +} + +pub struct EncryptedConnection { + connection: Connection, + encoder: CtrMode, + decoder: CtrMode, + mac_encoder: EcbEncryptor>, + egress_mac: Keccak, + ingress_mac: Keccak, + read_state: EncryptedConnectionState, + idle_timeout: Option, + protocol_id: u16, + payload_len: u32, +} + +impl EncryptedConnection { + pub fn new(handshake: Handshake) -> Result { + let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); + let mut nonce_material = H512::new(); + if handshake.originated { + handshake.remote_nonce.copy_to(&mut nonce_material[0..32]); + handshake.nonce.copy_to(&mut nonce_material[32..64]); + } + else { + handshake.nonce.copy_to(&mut nonce_material[0..32]); + handshake.remote_nonce.copy_to(&mut nonce_material[32..64]); + } + let mut key_material = H512::new(); + shared.copy_to(&mut key_material[0..32]); + nonce_material.sha3_into(&mut key_material[32..64]); + key_material.sha3().copy_to(&mut key_material[32..64]); + + let iv = vec![0u8; 16]; + let encoder = CtrMode::new(AesSafe128Encryptor::new(&key_material[32..64]), iv); + let iv = vec![0u8; 16]; + let decoder = CtrMode::new(AesSafe128Encryptor::new(&key_material[32..64]), iv); + + key_material.sha3().copy_to(&mut key_material[32..64]); + let mac_encoder = EcbEncryptor::new(AesSafe128Encryptor::new(&key_material[32..64]), NoPadding); + + let mut egress_mac = Keccak::new_keccak256(); + let mut mac_material = &H256::from_slice(&key_material[32..64]) ^ &handshake.remote_nonce; + egress_mac.update(&mac_material); + egress_mac.update(if handshake.originated { &handshake.auth_cipher } else { &handshake.ack_cipher }); + + let mut ingress_mac = Keccak::new_keccak256(); + mac_material = &(&mac_material ^ &handshake.remote_nonce) ^ &handshake.nonce; + ingress_mac.update(&mac_material); + ingress_mac.update(if handshake.originated { &handshake.ack_cipher } else { &handshake.auth_cipher }); + + Ok(EncryptedConnection { + connection: handshake.connection, + encoder: encoder, + decoder: decoder, + mac_encoder: mac_encoder, + egress_mac: egress_mac, + ingress_mac: ingress_mac, + read_state: EncryptedConnectionState::Header, + idle_timeout: None, + protocol_id: 0, + payload_len: 0 + }) + } + + pub fn write_packet(&mut self, payload: &[u8]) -> Result<(), Error> { + let mut header = RlpStream::new(); + let len = payload.len() as usize; + header.append_raw(&[(len >> 16) as u8, (len >> 8) as u8, len as u8], 1); + header.append_raw(&[0xc2u8, 0x80u8, 0x80u8], 1); + //TODO: ger rid of vectors here + let mut header = header.out(); + let padding = (16 - (payload.len() % 16)) % 16; + header.resize(16, 0u8); + + let mut packet = vec![0u8; (32 + payload.len() + padding + 16)]; + self.encoder.encrypt(&mut RefReadBuffer::new(&header), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding"); + self.egress_mac.update(&packet[0..16]); + self.egress_mac.clone().finalize(&mut packet[16..32]); + self.encoder.encrypt(&mut RefReadBuffer::new(&payload), &mut RefWriteBuffer::new(&mut packet[32..(32 + len)]), padding == 0).expect("Invalid length or padding"); + if padding != 0 { + let pad = [08; 16]; + self.encoder.encrypt(&mut RefReadBuffer::new(&pad[0..padding]), &mut RefWriteBuffer::new(&mut packet[(32 + len)..(32 + len + padding)]), true).expect("Invalid length or padding"); + } + self.egress_mac.update(&packet[32..(32 + len + padding)]); + self.egress_mac.clone().finalize(&mut packet[(32 + len + padding)..]); + self.connection.send(&packet); + Ok(()) + } + + fn read_header(&mut self, header: &[u8]) -> Result<(), Error> { + if header.len() != ENCRYPTED_HEADER_LEN { + return Err(Error::Auth); + } + self.ingress_mac.update(header); + let mac = &header[16..]; + let mut expected = H128::new(); + self.ingress_mac.clone().finalize(&mut expected); + if mac != &expected[..] { + return Err(Error::Auth); + } + + let mut header_dec = H128::new(); + self.decoder.decrypt(&mut RefReadBuffer::new(&header[0..16]), &mut RefWriteBuffer::new(&mut header_dec), false).expect("Invalid length or padding"); + + let length = ((header[0] as u32) << 8 + header[1] as u32) << 8 + header[2] as u32; + let header_rlp = UntrustedRlp::new(&header[3..]); + let protocol_id = try!(u16::decode_untrusted(&try!(header_rlp.at(0)))); + + self.payload_len = length; + self.protocol_id = protocol_id; + self.read_state = EncryptedConnectionState::Payload; + + let padding = (16 - (length % 16)) % 16; + let full_length = length + padding + 16; + self.connection.expect(full_length as usize); + Ok(()) + } + + fn read_payload(&mut self, payload: &[u8]) -> Result { + let padding = (16 - (self.payload_len % 16)) % 16; + let full_length = (self.payload_len + padding + 16) as usize; + if payload.len() != full_length { + return Err(Error::Auth); + } + self.ingress_mac.update(&payload[0..payload.len() - 16]); + let mac = &payload[(payload.len() - 16)..]; + let mut expected = H128::new(); + self.ingress_mac.clone().finalize(&mut expected); + if mac != &expected[..] { + return Err(Error::Auth); + } + + let mut packet = vec![0u8; self.payload_len as usize]; + self.decoder.decrypt(&mut RefReadBuffer::new(&payload[0..(full_length - 16)]), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding"); + packet.resize(self.payload_len as usize, 0u8); + Ok(packet) + } + + pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, Error> { + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + try!(self.connection.reregister(event_loop)); + match self.read_state { + EncryptedConnectionState::Header => { + match try!(self.connection.readable()) { + Some(data) => { + try!(self.read_header(&data)); + }, + None => {} + }; + Ok(None) + }, + EncryptedConnectionState::Payload => { + match try!(self.connection.readable()) { + Some(data) => { + self.read_state = EncryptedConnectionState::Header; + self.connection.expect(ENCRYPTED_HEADER_LEN); + Ok(Some(try!(self.read_payload(&data)))) + }, + None => Ok(None) + } + } + } + } + + pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + try!(self.connection.writable()); + try!(self.connection.reregister(event_loop)); + Ok(()) + } + + pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + self.connection.expect(ENCRYPTED_HEADER_LEN); + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); + try!(self.connection.register(event_loop)); + Ok(()) + } +} + impl Connection { pub fn new(token: Token, socket: TcpStream) -> Connection { Connection { @@ -35,11 +231,12 @@ impl Connection { if self.rec_size != self.rec_buf.len() { warn!(target:"net", "Unexpected connection read start"); } - unsafe { self.rec_buf.set_len(size) } + unsafe { self.rec_buf.set_len(0) } self.rec_size = size; } - pub fn readable(&mut self) -> io::Result> { + //TODO: return a slice + pub fn readable(&mut self) -> io::Result> { if self.rec_size == 0 || self.rec_buf.len() >= self.rec_size { warn!(target:"net", "Unexpected connection read"); } @@ -47,7 +244,10 @@ impl Connection { // resolve "multiple applicable items in scope [E0034]" error let sock_ref = ::by_ref(&mut self.socket); match sock_ref.take(max as u64).try_read_buf(&mut self.rec_buf) { - Ok(Some(_)) if self.rec_buf.len() == self.rec_size => Ok(Some(&self.rec_buf[0..self.rec_size])), + Ok(Some(_)) if self.rec_buf.len() == self.rec_size => { + self.rec_size = 0; + Ok(Some(::std::mem::replace(&mut self.rec_buf, Bytes::new()))) + }, Ok(_) => Ok(None), Err(e) => Err(e), } @@ -107,3 +307,4 @@ impl Connection { } } + diff --git a/src/network/handshake.rs b/src/network/handshake.rs index ef1cf418b..186d6b8a7 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -1,6 +1,7 @@ use mio::*; use mio::tcp::*; use hash::*; +use bytes::Bytes; use crypto::*; use crypto; use network::connection::{Connection, WriteStatus}; @@ -14,8 +15,6 @@ enum HandshakeState { WritingAuth, ReadingAck, WritingAck, - WritingHello, - ReadingHello, StartSession, } @@ -23,34 +22,46 @@ pub struct Handshake { pub id: NodeId, pub connection: Connection, state: HandshakeState, + pub originated: bool, idle_timeout: Option, - ecdhe: KeyPair, - nonce: H256, - remote_public: Public, - remote_nonce: H256 + pub ecdhe: KeyPair, + pub nonce: H256, + pub remote_public: Public, + pub remote_nonce: H256, + pub auth_cipher: Bytes, + pub ack_cipher: Bytes } +const AUTH_PACKET_SIZE:usize = 307; +const ACK_PACKET_SIZE:usize = 210; + impl Handshake { pub fn new(token: Token, id: &NodeId, socket: TcpStream, nonce: &H256) -> Result { Ok(Handshake { id: id.clone(), connection: Connection::new(token, socket), + originated: false, state: HandshakeState::New, idle_timeout: None, ecdhe: try!(KeyPair::create()), nonce: nonce.clone(), remote_public: Public::new(), - remote_nonce: H256::new() + remote_nonce: H256::new(), + auth_cipher: Bytes::new(), + ack_cipher: Bytes::new(), }) } - pub fn start(&mut self, host: &HostInfo, originated: bool) { + pub fn start(&mut self, host: &HostInfo, originated: bool) -> Result<(), Error> { + self.originated = originated; if originated { - self.write_auth(host); + try!(self.write_auth(host)); } else { - self.read_auth(); + self.state = HandshakeState::ReadingAuth; + self.connection.expect(AUTH_PACKET_SIZE); }; + Ok(()) } pub fn done(&self) -> bool { @@ -58,88 +69,138 @@ impl Handshake { } pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { - self.idle_timeout.map(|t| event_loop.clear_timeout(t)); - Ok(()) - } - - pub fn writable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); match self.state { - HandshakeState::WritingAuth => { - match (try!(self.connection.writable())) { - WriteStatus::Complete => { try!(self.read_ack()); }, - _ => {} + HandshakeState::ReadingAuth => { + match try!(self.connection.readable()) { + Some(data) => { + try!(self.read_auth(host, &data)); + try!(self.write_ack()); + }, + None => {} }; - try!(self.connection.reregister(event_loop)); }, - HandshakeState::WritingAck => { - match (try!(self.connection.writable())) { - WriteStatus::Complete => { try!(self.read_hello()); }, - _ => {} - }; - try!(self.connection.reregister(event_loop)); - }, - HandshakeState::WritingHello => { - match (try!(self.connection.writable())) { - WriteStatus::Complete => { self.state = HandshakeState::StartSession; }, - _ => { try!(self.connection.reregister(event_loop)); } + HandshakeState::ReadingAck => { + match try!(self.connection.readable()) { + Some(data) => { + try!(self.read_ack(host, &data)); + self.state = HandshakeState::StartSession; + }, + None => {} }; }, _ => { panic!("Unexpected state") } } + try!(self.connection.reregister(event_loop)); + Ok(()) + } + + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { + self.idle_timeout.map(|t| event_loop.clear_timeout(t)); + match self.state { + HandshakeState::WritingAuth => { + match try!(self.connection.writable()) { + WriteStatus::Complete => { + self.connection.expect(ACK_PACKET_SIZE); + self.state = HandshakeState::ReadingAck; + }, + _ => {} + }; + }, + HandshakeState::WritingAck => { + match try!(self.connection.writable()) { + WriteStatus::Complete => { + self.connection.expect(32); + self.state = HandshakeState::StartSession; + }, + _ => {} + }; + }, + _ => { panic!("Unexpected state") } + } + try!(self.connection.reregister(event_loop)); Ok(()) } pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); - self.connection.register(event_loop); + try!(self.connection.register(event_loop)); Ok(()) } - fn read_auth(&mut self) -> Result<(), Error> { - Ok(()) + fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), Error> { + trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); + assert!(data.len() == AUTH_PACKET_SIZE); + self.auth_cipher = data.to_vec(); + let auth = try!(ecies::decrypt(host.secret(), data)); + let (sig, rest) = auth.split_at(65); + let (hepubk, rest) = rest.split_at(32); + let (pubk, rest) = rest.split_at(64); + let (nonce, _) = rest.split_at(32); + self.remote_public.clone_from_slice(pubk); + self.remote_nonce.clone_from_slice(nonce); + let shared = try!(ecdh::agree(host.secret(), &self.remote_public)); + let signature = ec::Signature::from_slice(sig); + let spub = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce))); + if &spub.sha3()[..] != hepubk { + trace!(target:"net", "Handshake hash mismath with {:?}", self.connection.socket.peer_addr()); + return Err(Error::Auth); + }; + self.write_ack() } - fn read_ack(&mut self) -> Result<(), Error> { - Ok(()) - } - - fn read_hello(&mut self) -> Result<(), Error> { + fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), Error> { + trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); + assert!(data.len() == ACK_PACKET_SIZE); + self.ack_cipher = data.to_vec(); + let ack = try!(ecies::decrypt(host.secret(), data)); + let (pubk, nonce) = ack.split_at(65); + self.remote_public.clone_from_slice(pubk); + self.remote_nonce.clone_from_slice(nonce); Ok(()) } fn write_auth(&mut self, host: &HostInfo) -> Result<(), Error> { - trace!(target:"net", "Sending auth to {:?}", self.connection.socket.peer_addr()); + trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants let len = data.len(); { data[len - 1] = 0x0; let (sig, rest) = data.split_at_mut(65); let (hepubk, rest) = rest.split_at_mut(32); - let (mut pubk, rest) = rest.split_at_mut(64); - let (nonce, rest) = rest.split_at_mut(32); + let (pubk, rest) = rest.split_at_mut(64); + let (nonce, _) = rest.split_at_mut(32); // E(remote-pubk, S(ecdhe-random, ecdh-shared-secret^nonce) || H(ecdhe-random-pubk) || pubk || nonce || 0x0) let shared = try!(crypto::ecdh::agree(host.secret(), &self.id)); - let signature = try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig); + try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig); self.ecdhe.public().sha3_into(hepubk); - host.id().copy_to(&mut pubk); + host.id().copy_to(pubk); self.nonce.copy_to(nonce); } let message = try!(crypto::ecies::encrypt(&self.id, &data)); self.connection.send(&message[..]); + self.auth_cipher = message; self.state = HandshakeState::WritingAuth; Ok(()) } fn write_ack(&mut self) -> Result<(), Error> { + trace!(target:"net", "Sending handshake ack to {:?}", self.connection.socket.peer_addr()); + let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants + let len = data.len(); + { + data[len - 1] = 0x0; + let (epubk, rest) = data.split_at_mut(64); + let (nonce, _) = rest.split_at_mut(32); + self.ecdhe.public().copy_to(epubk); + self.nonce.copy_to(nonce); + } + let message = try!(crypto::ecies::encrypt(&self.id, &data)); + self.connection.send(&message[..]); + self.ack_cipher = message; + self.state = HandshakeState::WritingAck; Ok(()) } - - fn write_hello(&mut self) -> Result<(), Error> { - Ok(()) - } - - } diff --git a/src/network/host.rs b/src/network/host.rs index d436fae5f..cdb6dd132 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] //TODO: remove this after everything is done //TODO: remove all unwraps -use std::net::{SocketAddr, AddrParseError}; +use std::net::{SocketAddr, ToSocketAddrs}; use std::collections::{HashSet, HashMap, BTreeMap}; use std::hash::{Hash, Hasher}; use std::cell::{RefCell}; @@ -13,9 +13,9 @@ use mio::udp::*; use hash::*; use crypto::*; use time::Tm; -use error::EthcoreError; -use network::connection::Connection; use network::handshake::Handshake; +use network::session::Session; +use network::Error; const DEFAULT_PORT: u16 = 30303; @@ -65,32 +65,20 @@ impl NodeEndpoint { udp_port: address.port() } } - fn from_str(address: &str) -> Result { - let address = try!(SocketAddr::from_str(address)); - Ok(NodeEndpoint { - address: address, - udp_port: address.port() - }) + fn from_str(s: &str) -> Result { + println!("{:?}", s); + let address = s.to_socket_addrs().map(|mut i| i.next()); + match address { + Ok(Some(a)) => Ok(NodeEndpoint { + address: a, + udp_port: a.port() + }), + Ok(_) => Err(Error::AddressResolve(None)), + Err(e) => Err(Error::AddressResolve(Some(e))) + } } } -#[derive(Debug)] -pub enum AddressError { - AddrParseError(AddrParseError), - NodeIdParseError(EthcoreError) -} - -impl From for AddressError { - fn from(err: AddrParseError) -> AddressError { - AddressError::AddrParseError(err) - } -} -impl From for AddressError { - fn from(err: EthcoreError) -> AddressError { - AddressError::NodeIdParseError(err) - } -} - #[derive(PartialEq, Eq, Copy, Clone)] enum PeerType { Required, @@ -106,10 +94,10 @@ struct Node { } impl FromStr for Node { - type Err = AddressError; + type Err = Error; fn from_str(s: &str) -> Result { - let (id, endpoint) = if &s[..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { - (try!(NodeId::from_str(&s[8..128])), try!(NodeEndpoint::from_str(&s[137..]))) + let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { + (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) } else { (NodeId::new(), try!(NodeEndpoint::from_str(s))) @@ -164,11 +152,6 @@ impl NodeBucket { } } -struct Peer { - id: NodeId, - connection: Connection, -} - struct FindNodePacket; impl FindNodePacket { @@ -190,8 +173,6 @@ const NODETABLE_MAINTAIN: usize = 5; const NODETABLE_DISCOVERY: usize = 6; const FIRST_CONNECTION: usize = 7; const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; -const FIRST_HANDSHAKE: usize = FIRST_CONNECTION + MAX_CONNECTIONS; -const LAST_HANDSHAKE: usize = FIRST_HANDSHAKE + MAX_CONNECTIONS - 1; pub enum HostMessage { Shutdown @@ -217,13 +198,17 @@ impl HostInfo { } } +enum ConnectionEntry { + Handshake(Handshake), + Session(Session) +} + pub struct Host { info: HostInfo, sender: Sender, udp_socket: UdpSocket, listener: TcpListener, - peers: Slab, - connecting: Slab, + connections: Slab, discovery_round: u16, discovery_id: NodeId, discovery_nodes: HashSet, @@ -269,8 +254,7 @@ impl Host { sender: sender, udp_socket: udp_socket, listener: listener, - peers: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), - connecting: Slab::new_starting_at(Token(FIRST_HANDSHAKE), MAX_CONNECTIONS), + connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), discovery_round: 0, discovery_id: NodeId::new(), discovery_nodes: HashSet::new(), @@ -281,10 +265,10 @@ impl Host { host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); - host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02dd@gav.ethdev.com:30303"); - host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163@52.16.188.185:30303"); - host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e03@54.207.93.166:30303"); - host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254@92.51.165.126:30303"); + host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); + host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); + host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); + host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); event_loop.run(&mut host).unwrap(); } @@ -299,7 +283,10 @@ impl Host { fn add_node(&mut self, id: &str) { match Node::from_str(id) { Err(e) => { warn!("Could not add node: {:?}", e); }, - Ok(n) => { self.nodes.insert(n.id.clone(), n); } + Ok(n) => { + self.node_buckets[Host::distance(self.info.id(), &n.id) as usize].nodes.push(n.id.clone()); + self.nodes.insert(n.id.clone(), n); + } } } @@ -343,13 +330,7 @@ impl Host { } fn distance(a: &NodeId, b: &NodeId) -> u32 { - //TODO: - //u256 d = sha3(_a) ^ sha3(_b); - let mut d: NodeId = NodeId::new(); - for i in 0..32 { - d[i] = a[i] ^ b[i]; - } - + let d = a.sha3() ^ b.sha3(); let mut ret:u32 = 0; for i in 0..32 { let mut v: u8 = d[i]; @@ -448,11 +429,11 @@ impl Host { } fn have_session(&self, id: &NodeId) -> bool { - self.peers.iter().any(|h| h.id.eq(&id)) + self.connections.iter().any(|e| match e { &ConnectionEntry::Session(ref s) => s.id.eq(&id), _ => false }) } fn connecting_to(&self, id: &NodeId) -> bool { - self.connecting.iter().any(|h| h.id.eq(&id)) + self.connections.iter().any(|e| match e { &ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) } fn connect_peers(&mut self, event_loop: &mut EventLoop) { @@ -529,10 +510,18 @@ impl Host { }; let nonce = self.info.next_nonce(); - match self.connecting.insert_with(|token| Handshake::new(token, id, socket, &nonce).expect("Can't create handshake")) { + match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(token, id, socket, &nonce).expect("Can't create handshake"))) { Some(token) => { - self.connecting[token].register(event_loop).expect("Handshake token regisration failed"); - self.connecting[token].start(&self.info, true); + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Handshake(ref mut h)) => { + h.start(&self.info, true) + .and_then(|_| h.register(event_loop)) + .unwrap_or_else (|e| { + debug!(target: "net", "Handshake create error: {:?}", e); + }); + }, + _ => {} + } }, None => { warn!("Max connections reached") } } @@ -543,57 +532,86 @@ impl Host { warn!(target: "net", "accept"); } - fn handshake_writable(&mut self, token: Token, event_loop: &mut EventLoop) { - if !{ - let handshake = match self.connecting.get_mut(token) { - Some(h) => h, - None => { - warn!(target: "net", "Received event for unknown handshake"); - return; + fn connection_writable(&mut self, token: Token, event_loop: &mut EventLoop) { + let mut kill = false; + let mut create_session = false; + { + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Handshake(ref mut h)) => { + h.writable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Handshake write error: {:?}", e); + kill = true; + }); + create_session = h.done(); + }, + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.writable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Session write error: {:?}", e); + kill = true; + }); + } + _ => { + warn!(target: "net", "Received event for unknown connection"); } }; - match handshake.writable(event_loop, &self.info) { - Err(e) => { - debug!(target: "net", "Handshake read error: {:?}", e); - false - }, - Ok(_) => true - } - } { - self.kill_handshake(token, event_loop); + } + if kill { + self.kill_connection(token, event_loop); + } + if create_session { + self.start_session(token, event_loop); } } - fn handshake_readable(&mut self, token: Token, event_loop: &mut EventLoop) { - if !{ - let handshake = match self.connecting.get_mut(token) { - Some(h) => h, - None => { - warn!(target: "net", "Received event for unknown handshake"); - return; + fn connection_readable(&mut self, token: Token, event_loop: &mut EventLoop) { + let mut kill = false; + let mut create_session = false; + { + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Handshake(ref mut h)) => { + h.readable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Handshake read error: {:?}", e); + kill = true; + }); + create_session = h.done(); + }, + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.readable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Session read error: {:?}", e); + kill = true; + }); + } + _ => { + warn!(target: "net", "Received event for unknown connection"); } }; - match handshake.writable(event_loop, &self.info) { - Err(e) => { - debug!(target: "net", "Handshake read error: {:?}", e); - false - }, - Ok(_) => true - } - } { - self.kill_handshake(token, event_loop); + } + if kill { + self.kill_connection(token, event_loop); + } + if create_session { + self.start_session(token, event_loop); } } - fn handshake_timeout(&mut self, token: Token, event_loop: &mut EventLoop) { - self.kill_handshake(token, event_loop) - } - fn kill_handshake(&mut self, token: Token, _event_loop: &mut EventLoop) { - self.connecting.remove(token); + + fn start_session(&mut self, token: Token, event_loop: &mut EventLoop) { + self.connections.replace_with(token, |c| { + match c { + ConnectionEntry::Handshake(h) => Session::new(h, event_loop) + .map(|s| Some(ConnectionEntry::Session(s))) + .unwrap_or_else(|e| { + debug!(target: "net", "Session construction error: {:?}", e); + None + }), + _ => { panic!("No handshake to create a session from"); } + } + }).expect("Error updating slab with session"); } - fn read_connection(&mut self, _token: Token, _event_loop: &mut EventLoop) { + fn connection_timeout(&mut self, token: Token, event_loop: &mut EventLoop) { + self.kill_connection(token, event_loop) } - - fn write_connection(&mut self, _token: Token, _event_loop: &mut EventLoop) { + fn kill_connection(&mut self, token: Token, _event_loop: &mut EventLoop) { + self.connections.remove(token); } } @@ -606,16 +624,14 @@ impl Handler for Host { match token.as_usize() { TCP_ACCEPT => self.accept(event_loop), IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.read_connection(token, event_loop), - FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_readable(token, event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), NODETABLE_RECEIVE => {}, _ => panic!("Received unknown readable token"), } } else if events.is_writable() { match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.write_connection(token, event_loop), - FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_writable(token, event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), _ => panic!("Received unknown writable token"), } } @@ -624,7 +640,7 @@ impl Handler for Host { fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { match token.as_usize() { IDLE => self.maintain_network(event_loop), - FIRST_HANDSHAKE ... LAST_HANDSHAKE => self.handshake_timeout(token, event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, _ => panic!("Received unknown timer token"), @@ -636,9 +652,11 @@ impl Handler for Host { #[cfg(test)] mod tests { use network::host::Host; + use env_logger; #[test] - #[ignore] + //#[ignore] fn net_connect() { + env_logger::init().unwrap(); let _ = Host::start(); } } diff --git a/src/network/mod.rs b/src/network/mod.rs index b8682dd12..d1f9940c4 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,13 +1,18 @@ extern crate mio; -pub mod host; -pub mod connection; -pub mod handshake; - +mod host; +mod connection; +mod handshake; +mod session; #[derive(Debug)] pub enum Error { Crypto(::crypto::CryptoError), Io(::std::io::Error), + Auth, + BadProtocol, + AddressParse(::std::net::AddrParseError), + AddressResolve(Option<::std::io::Error>), + NodeIdParse(::error::EthcoreError), } impl From<::std::io::Error> for Error { @@ -21,4 +26,23 @@ impl From<::crypto::CryptoError> for Error { Error::Crypto(err) } } +impl From<::std::net::AddrParseError> for Error { + fn from(err: ::std::net::AddrParseError) -> Error { + Error::AddressParse(err) + } +} +impl From<::error::EthcoreError> for Error { + fn from(err: ::error::EthcoreError) -> Error { + Error::NodeIdParse(err) + } +} +impl From<::rlp::DecoderError> for Error { + fn from(_err: ::rlp::DecoderError) -> Error { + Error::Auth + } +} +pub fn start_host() +{ + let _ = host::Host::start(); +} diff --git a/src/network/session.rs b/src/network/session.rs new file mode 100644 index 000000000..71554b0ca --- /dev/null +++ b/src/network/session.rs @@ -0,0 +1,33 @@ +#![allow(dead_code)] //TODO: remove this after everything is done +//TODO: remove all unwraps +use mio::*; +use hash::*; +use network::connection::{EncryptedConnection}; +use network::handshake::Handshake; +use network::Error; +use network::host::*; + +pub struct Session { + pub id: NodeId, + connection: EncryptedConnection, +} + +impl Session { + pub fn new(h: Handshake, event_loop: &mut EventLoop) -> Result { + let id = h.id.clone(); + let mut connection = try!(EncryptedConnection::new(h)); + try!(connection.register(event_loop)); + Ok(Session { + id: id, + connection: connection, + }) + } + pub fn readable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { + try!(self.connection.readable(event_loop)); + Ok(()) + } + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { + self.connection.writable(event_loop) + } +} + diff --git a/src/sha3.rs b/src/sha3.rs index a8b5c7c44..8ceedb15f 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -19,7 +19,9 @@ impl Hashable for T where T: BytesConvertable { } } fn sha3_into(&self, dest: &mut [u8]) { - keccak_256(self.bytes(), dest); + let mut keccak = Keccak::new_keccak256(); + keccak.update(self.bytes()); + keccak.finalize(dest); } } From 18a6090ce5ac9fb72934d80731ca429a95557b76 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 12:57:25 +0100 Subject: [PATCH 182/379] fixed trie_hash panic when he was processing duplicates --- src/triehash.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 33d35513c..56dafcb2e 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -32,12 +32,12 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { .into_iter() .fold(BTreeMap::new(), | mut acc, vec | { let len = acc.len(); - acc.insert(as_nibbles(&rlp::encode(&len)), vec); + acc.insert(rlp::encode(&len), vec); acc }) // then move them to a vector .into_iter() - .map(|p| p ) + .map(|(k, v)| (as_nibbles(&k), v) ) .collect(); gen_trie_root(gen_input) @@ -62,11 +62,17 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { /// assert_eq!(trie_root(v), H256::from_str(root).unwrap()); /// } /// ``` -pub fn trie_root(mut input: Vec<(Vec, Vec)>) -> H256 { - input.sort(); +pub fn trie_root(input: Vec<(Vec, Vec)>) -> H256 { let gen_input = input + // first put elements into btree to sort them and to remove duplicates .into_iter() - .map(|(k, v)| (as_nibbles(&k), v)) + .fold(BTreeMap::new(), | mut acc, (k, v) | { + acc.insert(k, v); + acc + }) + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v) ) .collect(); gen_trie_root(gen_input) @@ -139,6 +145,7 @@ fn as_nibbles(bytes: &[u8]) -> Vec { fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { let inlen = input.len(); + //println!("input: {:?}", input); // in case of empty slice, just append empty data if inlen == 0 { stream.append_empty_data(); @@ -167,6 +174,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre cmp::min(key.shared_prefix_len(&k), acc) }); + println!("shared_prefix: {}, prefix_len: {}", shared_prefix, pre_len); // if shared prefix is higher than current prefix append its // new part of the key to the stream // then recursively append suffixes of all items who had this key @@ -192,8 +200,8 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // cout how many successive elements have same next nibble let len = match begin < input.len() { true => input[begin..].iter() - .map(| pair | pair.0[pre_len] ) - .take_while(|&q| q == i).count(), + .take_while(| pair | { println!("{:?}", pair.0); pair.0[pre_len] == i }).count(), + //.take_while(|&q| q == i).count(), false => 0 }; From 4acb3dd727162f56685a475955f1f1b97f09d1d4 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 13:07:21 +0100 Subject: [PATCH 183/379] tests submodule --- .gitmodules | 3 +++ tests | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 tests diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..2571e36c6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tests"] + path = tests + url = https://github.com/ethereum/tests diff --git a/tests b/tests new file mode 160000 index 000000000..2e4987ad2 --- /dev/null +++ b/tests @@ -0,0 +1 @@ +Subproject commit 2e4987ad2a973e2cf85ef742a8b9bd094363cd18 From 21f87cfb0f92754d2833444e50d853fa86e77007 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 13:58:29 +0100 Subject: [PATCH 184/379] ItemInfo -> PayloadInfo --- src/rlp.rs | 49 ++++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index c023b1dfb..1648f8807 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -65,16 +65,16 @@ impl OffsetCache { } } -/// stores basic information about item -struct ItemInfo { - prefix_len: usize, - value_len: usize, +/// Stores basic information about item +pub struct PayloadInfo { + pub header_len: usize, + pub value_len: usize, } -impl ItemInfo { - fn new(prefix_len: usize, value_len: usize) -> ItemInfo { - ItemInfo { - prefix_len: prefix_len, +impl PayloadInfo { + fn new(header_len: usize, value_len: usize) -> PayloadInfo { + PayloadInfo { + header_len: header_len, value_len: value_len, } } @@ -356,7 +356,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { pub fn data(&'view self) -> &'a [u8] { let ii = Self::item_info(self.bytes).unwrap(); - &self.bytes[ii.prefix_len..(ii.prefix_len + ii.value_len)] + &self.bytes[ii.header_len..(ii.header_len + ii.value_len)] } /// Returns number of rlp items. @@ -401,6 +401,8 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } } + //pub fn payload_offset(&self) -> + /// Get view onto rlp-slice at index /// /// Caches offset to given index, so access to successive @@ -438,7 +440,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { // construct new rlp let found = try!(UntrustedRlp::item_info(bytes)); - Ok(UntrustedRlp::new(&bytes[0..found.prefix_len + found.value_len])) + Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) } /// No value @@ -553,7 +555,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// consumes first found prefix fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { let item = try!(UntrustedRlp::item_info(self.bytes)); - let bytes = try!(UntrustedRlp::consume(self.bytes, item.prefix_len)); + let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); Ok(bytes) } @@ -562,7 +564,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { let mut result = bytes; for _ in 0..items { let i = try!(UntrustedRlp::item_info(result)); - result = try!(UntrustedRlp::consume(result, (i.prefix_len + i.value_len))); + result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); } Ok(result) } @@ -570,28 +572,29 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// return first item info /// /// TODO: move this to decoder (?) - fn item_info(bytes: &[u8]) -> Result { + fn item_info(bytes: &[u8]) -> Result { let item = match bytes.first().map(|&x| x) { None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => ItemInfo::new(0, 1), - Some(l @ 0x80...0xb7) => ItemInfo::new(1, l as usize - 0x80), + Some(0...0x7f) => PayloadInfo::new(0, 1), + Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), Some(l @ 0xb8...0xbf) => { let len_of_len = l as usize - 0xb7; - let prefix_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); - ItemInfo::new(prefix_len, value_len) + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) } - Some(l @ 0xc0...0xf7) => ItemInfo::new(1, l as usize - 0xc0), + Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), Some(l @ 0xf8...0xff) => { let len_of_len = l as usize - 0xf7; - let prefix_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..prefix_len])); - ItemInfo::new(prefix_len, value_len) + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) } + // we cant reach this place, but rust requires _ to be implemented _ => return Err(DecoderError::BadRlp), }; - match item.prefix_len + item.value_len <= bytes.len() { + match item.header_len + item.value_len <= bytes.len() { true => Ok(item), false => Err(DecoderError::RlpIsTooShort), } From 35ae32a8a5b85724f66af20208f9f908fca882ae Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 14:05:17 +0100 Subject: [PATCH 185/379] payload_info moved to decoder --- src/rlp.rs | 71 +++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 1648f8807..67fad0bb0 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -355,7 +355,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } pub fn data(&'view self) -> &'a [u8] { - let ii = Self::item_info(self.bytes).unwrap(); + let ii = BasicDecoder::payload_info(self.bytes).unwrap(); &self.bytes[ii.header_len..(ii.header_len + ii.value_len)] } @@ -396,7 +396,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// ``` pub fn size(&self) -> usize { match self.is_data() { - true => Self::item_info(self.bytes).unwrap().value_len, + true => BasicDecoder::payload_info(self.bytes).unwrap().value_len, false => 0 } } @@ -439,7 +439,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); // construct new rlp - let found = try!(UntrustedRlp::item_info(bytes)); + let found = try!(BasicDecoder::payload_info(bytes)); Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) } @@ -554,7 +554,7 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { /// consumes first found prefix fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { - let item = try!(UntrustedRlp::item_info(self.bytes)); + let item = try!(BasicDecoder::payload_info(self.bytes)); let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); Ok(bytes) } @@ -563,42 +563,12 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { let mut result = bytes; for _ in 0..items { - let i = try!(UntrustedRlp::item_info(result)); + let i = try!(BasicDecoder::payload_info(result)); result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); } Ok(result) } - /// return first item info - /// - /// TODO: move this to decoder (?) - fn item_info(bytes: &[u8]) -> Result { - let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => PayloadInfo::new(0, 1), - Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - } - Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), - Some(l @ 0xf8...0xff) => { - let len_of_len = l as usize - 0xf7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - } - // we cant reach this place, but rust requires _ to be implemented - _ => return Err(DecoderError::BadRlp), - }; - - match item.header_len + item.value_len <= bytes.len() { - true => Ok(item), - false => Err(DecoderError::RlpIsTooShort), - } - } /// consumes slice prefix of length `len` fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { @@ -730,6 +700,37 @@ pub trait Decoder { struct BasicDecoder; +impl BasicDecoder { + /// Return first item info + fn payload_info(bytes: &[u8]) -> Result { + let item = match bytes.first().map(|&x| x) { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => PayloadInfo::new(0, 1), + Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + } + Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + } + // we cant reach this place, but rust requires _ to be implemented + _ => return Err(DecoderError::BadRlp), + }; + + match item.header_len + item.value_len <= bytes.len() { + true => Ok(item), + false => Err(DecoderError::RlpIsTooShort), + } + } +} + impl Decoder for BasicDecoder { fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { match bytes.first().map(|&x| x) { From 356be1c635d8817c0ffe54755e8a62d1a5aa7ff6 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 14:14:12 +0100 Subject: [PATCH 186/379] rlp payload_info method --- src/rlp.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 67fad0bb0..4c22b7dcc 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -170,6 +170,10 @@ impl<'a, 'view> Rlp<'a> where 'a: 'view { self.rlp.raw() } + pub fn payload_info(&self) -> PayloadInfo { + self.rlp.payload_info().unwrap() + } + pub fn data(&'view self) -> &'a [u8] { self.rlp.data() } @@ -354,6 +358,10 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { self.bytes } + pub fn payload_info(&self) -> Result { + BasicDecoder::payload_info(self.bytes) + } + pub fn data(&'view self) -> &'a [u8] { let ii = BasicDecoder::payload_info(self.bytes).unwrap(); &self.bytes[ii.header_len..(ii.header_len + ii.value_len)] @@ -401,8 +409,6 @@ impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { } } - //pub fn payload_offset(&self) -> - /// Get view onto rlp-slice at index /// /// Caches offset to given index, so access to successive From aa02a71d09a91f35274bb1d28db6ffbe5774a57f Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 14:20:41 +0100 Subject: [PATCH 187/379] removed DecoderError::BadRlp --- src/rlp.rs | 8 ++++---- src/triehash.rs | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 4c22b7dcc..5f718ac3e 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -86,7 +86,6 @@ pub enum DecoderError { RlpIsTooShort, RlpExpectedToBeList, RlpExpectedToBeData, - BadRlp, } impl StdError for DecoderError { fn description(&self) -> &str { @@ -725,9 +724,9 @@ impl BasicDecoder { let header_len = 1 + len_of_len; let value_len = try!(usize::from_bytes(&bytes[1..header_len])); PayloadInfo::new(header_len, value_len) - } + }, // we cant reach this place, but rust requires _ to be implemented - _ => return Err(DecoderError::BadRlp), + _ => { panic!(); } }; match item.header_len + item.value_len <= bytes.len() { @@ -753,7 +752,8 @@ impl Decoder for BasicDecoder { let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } - _ => Err(DecoderError::BadRlp), + // we are reading value, not a list! + _ => { panic!(); } } } } diff --git a/src/triehash.rs b/src/triehash.rs index 56dafcb2e..8c8e9fd85 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -280,7 +280,10 @@ fn test_hex_prefix_encode() { #[cfg(test)] mod tests { use std::str::FromStr; + use std::collections::BTreeMap; use rustc_serialize::hex::FromHex; + use rustc_serialize::json::Json; + use bytes::*; use hash::*; use triehash::*; @@ -377,4 +380,19 @@ mod tests { assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); } + + #[test] + fn test_triehash_json_trietest_json() { + let data = include_bytes!("../tests/TrieTests/trietest.json"); + + let s = String::from_bytes(data).unwrap(); + let json = Json::from_str(&s).unwrap(); + let obj = json.as_object().unwrap(); + + for (key, value) in obj.iter() { + println!("running test: {}", key); + } + assert!(false); + } + } From b8c1447c0415b6f5d78c3bf17d75dbdeae796d5b Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 14:29:09 +0100 Subject: [PATCH 188/379] removed fmt::Debug from append --- src/rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp.rs b/src/rlp.rs index 5f718ac3e..b055632b1 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -810,7 +810,7 @@ impl RlpStream { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable + fmt::Debug { + pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { // encode given value and add it at the end of the stream object.encode(&mut self.encoder); From 65d49f280c7b10765c778fcbbc4bcf0900dbd901 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 15:03:20 +0100 Subject: [PATCH 189/379] Trie fix. --- src/trie.rs | 142 +++++++++++++++++++++++++++++++----------------- src/triehash.rs | 4 +- 2 files changed, 94 insertions(+), 52 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 2fb05ccfb..e140b851c 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -27,10 +27,10 @@ pub trait Trie { #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { - NullRoot, + Empty, Leaf(NibbleSlice<'a>, &'a[u8]), Extension(NibbleSlice<'a>, &'a[u8]), - Branch([Option<&'a[u8]>; 16], Option<&'a [u8]>) + Branch([&'a[u8]; 16], Option<&'a [u8]>) } impl <'a>Node<'a> { @@ -48,14 +48,14 @@ impl <'a>Node<'a> { }, // branch - first 16 are nodes, 17th is a value (or empty). Prototype::List(17) => { - let mut nodes: [Option<&'a [u8]>; 16] = unsafe { ::std::mem::uninitialized() }; + let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; for i in 0..16 { - nodes[i] = if r.at(i).is_empty() { None } else { Some(r.at(i).raw()) } + nodes[i] = r.at(i).raw(); } Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) }, // an empty branch index. - Prototype::Data(0) => Node::NullRoot, + Prototype::Data(0) => Node::Empty, // something went wrong. _ => panic!("Rlp is not valid.") } @@ -78,11 +78,8 @@ impl <'a>Node<'a> { }, Node::Branch(ref nodes, ref value) => { let mut stream = RlpStream::new_list(17); - for i in 0..16 { - match nodes[i] { - Some(n) => { stream.append_raw(n, 1); }, - None => { stream.append_empty_data(); }, - } + for i in 0..16 { + stream.append_raw(nodes[i], 1); } match *value { Some(n) => { stream.append(&n); }, @@ -90,7 +87,7 @@ impl <'a>Node<'a> { } stream.out() }, - Node::NullRoot => { + Node::Empty => { let mut stream = RlpStream::new(); stream.append_empty_data(); stream.out() @@ -155,7 +152,7 @@ impl fmt::Debug for TrieDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "[")); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - try!(self.fmt_all(root_rlp, f, 0)); + try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); writeln!(f, "]") } } @@ -198,13 +195,16 @@ impl TrieDB { Ok(()) } - fn fmt_all(&self, node: &[u8], f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { - let node = Node::decoded(node); + fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { + Node::decoded(self.get_raw_or_lookup(node)) + } + + fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { match node { Node::Leaf(slice, value) => try!(writeln!(f, "-{:?}: {:?}.", slice, value.pretty())), Node::Extension(ref slice, ref item) => { try!(write!(f, "-{:?}- ", slice)); - try!(self.fmt_all(self.get_raw_or_lookup(item), f, deepness)); + try!(self.fmt_all(self.get_node(item), f, deepness)); }, Node::Branch(ref nodes, ref value) => { try!(writeln!(f, "")); @@ -216,18 +216,18 @@ impl TrieDB { &None => {} } for i in 0..16 { - match nodes[i] { - Some(n) => { + match self.get_node(nodes[i]) { + Node::Empty => {}, + n => { try!(self.fmt_indent(f, deepness + 1)); try!(write!(f, "{:x}: ", i)); - try!(self.fmt_all(self.get_raw_or_lookup(n), f, deepness + 1)); - }, - None => {}, + try!(self.fmt_all(n, f, deepness + 1)); + } } } }, // empty - Node::NullRoot => { + Node::Empty => { try!(writeln!(f, "")); } }; @@ -247,12 +247,7 @@ impl TrieDB { }, Node::Branch(ref nodes, value) => match key.is_empty() { true => value, - false => match nodes[key.at(0) as usize] { - Some(payload) => { - self.get_from_node(self.get_raw_or_lookup(payload), &key.mid(1)) - }, - None => None - } + false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1)) }, _ => None } @@ -277,6 +272,16 @@ impl TrieDB { trace!("/"); } + fn delete(&mut self, key: &NibbleSlice) { + trace!("DELETE: {:?}", key); + // determine what the new root is, insert new nodes and remove old as necessary. + let mut todo: Diff = Diff::new(); + let root_rlp = self.cleared(self.db.lookup(&self.root).expect("Trie root not found!"), key, &mut todo); + self.apply(todo); + self.set_root_rlp(&root_rlp); + trace!("/"); + } + fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { trace!("compose_leaf {:?} {:?} ({:?})", partial, value.pretty(), partial.encoded(true).pretty()); let mut s = RlpStream::new_list(2); @@ -288,12 +293,12 @@ impl TrieDB { } fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { - println!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf)); + trace!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); s.append(&partial.encoded(is_leaf)); s.append_raw(raw_payload, 1); let r = s.out(); - println!("compose_raw: -> {:?}", r.pretty()); + trace!("compose_raw: -> {:?}", r.pretty()); r } @@ -361,17 +366,18 @@ impl TrieDB { s.out() } - fn transmuted_leaf_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { + fn transmuted_leaf_to_branch(orig_partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { trace!("transmuted_leaf_to_branch"); let mut s = RlpStream::new_list(17); let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; // orig is leaf - orig_raw_payload is data representing the actual value. for i in 0..17 { - if index == i { - // this is our node. - diff.new_node(Self::compose_raw(&orig_partial.mid(if i == 16 {0} else {1}), orig_raw_payload, true), &mut s); - } else { - s.append_empty_data(); + match (index == i, i) { + (true, 16) => // leaf entry - just replace. + { s.append(&value); }, + (true, _) => // easy - original had empty slot. + diff.new_node(Self::compose_leaf(&orig_partial.mid(1), value), &mut s), + (false, _) => { s.append_empty_data(); } } } s.out() @@ -383,7 +389,7 @@ impl TrieDB { fn transmuted_to_branch_and_augmented(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { trace!("transmuted_to_branch_and_augmented"); let intermediate = match orig_is_leaf { - true => Self::transmuted_leaf_to_branch(orig_partial, orig_raw_payload, diff), + true => Self::transmuted_leaf_to_branch(orig_partial, Rlp::new(orig_raw_payload).data(), diff), false => Self::transmuted_extension_to_branch(orig_partial, orig_raw_payload, diff), }; self.augmented(&intermediate, partial, value, diff) @@ -420,7 +426,7 @@ impl TrieDB { /// The database will be updated so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. /// - /// **This operation will not insert the new node now destroy the original.** + /// **This operation will not insert the new node nor destroy the original.** fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { trace!("augmented (old: {:?}, partial: {:?}, value: {:?})", old.pretty(), partial, value.pretty()); // already have an extension. either fast_forward, cleve or transmute_to_branch. @@ -486,6 +492,25 @@ impl TrieDB { _ => panic!("Invalid RLP for node: {:?}", old.pretty()), } } + + + /// Determine the RLP of the node, assuming we're removing `partial` from the + /// node currently of data `old`. This will *not* delete any hash of `old` from the database; + /// it will just return the new RLP that represents the new node. + /// + /// The database will be updated so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn cleared(&self, old: &[u8], partial: &NibbleSlice, _diff: &mut Diff) -> Bytes { + trace!("cleared (old: {:?}, partial: {:?})", old.pretty(), partial); + + unimplemented!(); + +/* match (Node::decoded(old)) { + + }*/ + } } impl Trie for TrieDB { @@ -503,8 +528,8 @@ impl Trie for TrieDB { self.add(&NibbleSlice::new(key), value); } - fn remove(&mut self, _key: &[u8]) { - unimplemented!(); + fn remove(&mut self, key: &[u8]) { + self.delete(&NibbleSlice::new(key)); } } @@ -512,12 +537,12 @@ impl Trie for TrieDB { mod tests { use rustc_serialize::hex::FromHex; use triehash::*; - use hash::*; use super::*; use nibbleslice::*; use rlp; use env_logger; use rand::random; + use bytes::ToPretty; #[test] fn test_node_leaf() { @@ -546,7 +571,7 @@ mod tests { #[test] fn test_node_empty_branch() { - let branch = Node::Branch([None; 16], None); + let branch = Node::Branch([&b""[..]; 16], None); let rlp = branch.encoded(); let branch2 = Node::decoded(&rlp); assert_eq!(branch, branch2); @@ -555,8 +580,8 @@ mod tests { #[test] fn test_node_branch() { let k = rlp::encode(&"cat"); - let mut nodes: [Option<&[u8]>; 16] = unsafe { ::std::mem::uninitialized() }; - for i in 0..16 { nodes[i] = Some(&k); } + let mut nodes: [&[u8]; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { nodes[i] = &k; } let v: Vec = From::from("dog"); let branch = Node::Branch(nodes, Some(&v)); let rlp = branch.encoded(); @@ -631,20 +656,37 @@ mod tests { #[test] fn stress() { - for _ in 0..1000 { + for _ in 0..10000 { let mut x: Vec<(Vec, Vec)> = Vec::new(); - for j in 0..100u32 { + for j in 0..4u32 { let key = random_key(); x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); - - let mem = trie_root_mem(&x); - assert_eq!(mem, real); + let memtrie = trie_root_mem(&x); + let mut y = x.clone(); + y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); + let memtrie_sorted = trie_root_mem(&y); + if *memtrie.root() != real || *memtrie_sorted.root() != real { + println!("TRIE MISMATCH"); + println!(""); + println!("ORIGINAL... {:?}", memtrie.root()); + for i in x.iter() { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("{:?}", memtrie); + println!("SORTED... {:?}", memtrie_sorted.root()); + for i in y.iter() { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("{:?}", memtrie_sorted); + } + assert_eq!(*memtrie.root(), real); + assert_eq!(*memtrie_sorted.root(), real); } } - fn trie_root_mem(v: &Vec<(Vec, Vec)>) -> H256 { + fn trie_root_mem(v: &Vec<(Vec, Vec)>) -> TrieDB { let mut t = TrieDB::new_memory(); for i in 0..v.len() { @@ -653,7 +695,7 @@ mod tests { t.insert(&key, &val); } - t.root().clone() + t } #[test] diff --git a/src/triehash.rs b/src/triehash.rs index 56dafcb2e..c06cdb249 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -174,7 +174,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre cmp::min(key.shared_prefix_len(&k), acc) }); - println!("shared_prefix: {}, prefix_len: {}", shared_prefix, pre_len); +// println!("shared_prefix: {}, prefix_len: {}", shared_prefix, pre_len); // if shared prefix is higher than current prefix append its // new part of the key to the stream // then recursively append suffixes of all items who had this key @@ -200,7 +200,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // cout how many successive elements have same next nibble let len = match begin < input.len() { true => input[begin..].iter() - .take_while(| pair | { println!("{:?}", pair.0); pair.0[pre_len] == i }).count(), + .take_while(| pair | { /*println!("{:?}", pair.0);*/ pair.0[pre_len] == i }).count(), //.take_while(|&q| q == i).count(), false => 0 }; From 9650494311ccfb89d18beaddeae3fa191be52622 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 15:03:47 +0100 Subject: [PATCH 190/379] remove redundant prints --- src/triehash.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 8c8e9fd85..778dc7f17 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -145,7 +145,6 @@ fn as_nibbles(bytes: &[u8]) -> Vec { fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStream) { let inlen = input.len(); - //println!("input: {:?}", input); // in case of empty slice, just append empty data if inlen == 0 { stream.append_empty_data(); @@ -174,7 +173,6 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre cmp::min(key.shared_prefix_len(&k), acc) }); - println!("shared_prefix: {}, prefix_len: {}", shared_prefix, pre_len); // if shared prefix is higher than current prefix append its // new part of the key to the stream // then recursively append suffixes of all items who had this key @@ -200,8 +198,8 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // cout how many successive elements have same next nibble let len = match begin < input.len() { true => input[begin..].iter() - .take_while(| pair | { println!("{:?}", pair.0); pair.0[pre_len] == i }).count(), - //.take_while(|&q| q == i).count(), + .take_while(| pair | pair.0[pre_len] == i ) + .count(), false => 0 }; From c184758972f4b472afb717648964109b197cdefc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 15:04:34 +0100 Subject: [PATCH 191/379] Reduce time needed for stress test. --- src/trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index e140b851c..704db94d4 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -656,7 +656,7 @@ mod tests { #[test] fn stress() { - for _ in 0..10000 { + for _ in 0..5000 { let mut x: Vec<(Vec, Vec)> = Vec::new(); for j in 0..4u32 { let key = random_key(); From 4372a1cd55cc68d502789361d9ad3ae9bce57a1a Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 17:00:49 +0100 Subject: [PATCH 192/379] removed submodule --- .gitmodules | 3 --- tests | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 tests diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 2571e36c6..000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "tests"] - path = tests - url = https://github.com/ethereum/tests diff --git a/tests b/tests deleted file mode 160000 index 2e4987ad2..000000000 --- a/tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2e4987ad2a973e2cf85ef742a8b9bd094363cd18 From 9c85e4d305a569d1f7a23b5878dabb5b3ba2fad2 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 17:52:06 +0100 Subject: [PATCH 193/379] commented out tests --- src/triehash.rs | 86 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index 778dc7f17..ab0c75683 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -278,7 +278,7 @@ fn test_hex_prefix_encode() { #[cfg(test)] mod tests { use std::str::FromStr; - use std::collections::BTreeMap; + use std::collections::HashMap; use rustc_serialize::hex::FromHex; use rustc_serialize::json::Json; use bytes::*; @@ -379,18 +379,80 @@ mod tests { assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); } - #[test] - fn test_triehash_json_trietest_json() { - let data = include_bytes!("../tests/TrieTests/trietest.json"); + //fn hex_or_string(s: &str) -> Vec { + //match s.starts_with("0x") { + //true => s[2..].from_hex().unwrap(), + //false => From::from(s) + //} + //} + + //pub fn yield_json_trietest(data: &[u8], name: &str, insert: &mut I, remove: &mut R) -> H256 where I: FnMut(Vec, Vec), R: FnMut(Vec) { + ////let data = include_bytes!("../tests/TrieTests/trietest.json"); - let s = String::from_bytes(data).unwrap(); - let json = Json::from_str(&s).unwrap(); - let obj = json.as_object().unwrap(); + //let s = String::from_bytes(data).unwrap(); + //let json = Json::from_str(&s).unwrap(); + //let obj = json.as_object().unwrap(); + //println!("here"); + + //let value = &obj[name]; + + //let i = &value["in"]; + //let o = &value["root"]; + + //let root_str = o.as_string().unwrap(); + + //println!("here2"); + //for i in i.as_array().unwrap().iter() { + //let key = hex_or_string(i[0].as_string().unwrap()); + + //match i[1].is_null() { + //true => remove(key), + //false => insert(key, hex_or_string(i[1].as_string().unwrap())) + //}; + //} + + //H256::from_str(&root_str[2..]).unwrap() + //} + + ////fn load_json_trietest(data: &[u8], name: &str) -> (Vec<(Vec, Vec)>, H256) { + ////use std::cell::RefCell; + //let map = RefCell::new(HashMap::new()); + //let root = yield_json_trietest(data, name, &mut | key, value | { + //map.borrow_mut().insert(key, value); + //}, &mut | key | { + //map.borrow_mut().remove(&key); + //}); + + //let res = map.into_inner() + //.into_iter() + //.map(|p| p) + //.collect(); + //(res, root) + //} + + //#[test] + //fn triehash_json_empty_values() { + //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/trietest.json"), "emptyValues"); + //assert_eq!(trie_root(input), root); + //} + + //#[test] + //fn triehash_json_branching_tests() { + //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/trietest.json"), "branchingTests"); + //assert_eq!(trie_root(input), root); + //} + + //#[test] + //fn triehash_json_jeff_tests() { + //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/trietest.json"), "jeff"); + //assert_eq!(trie_root(input), root); + //} + + ////#[test] + ////fn triehash_json_test1() { + //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/hex_encoded_securetrie_test.json"), "test1"); + //assert_eq!(trie_root(input), root); + //} - for (key, value) in obj.iter() { - println!("running test: {}", key); - } - assert!(false); - } } From 75b1de04822cdbb7b87cfa05ae7084fd2090ee79 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 2 Dec 2015 20:11:13 +0100 Subject: [PATCH 194/379] Session --- src/network/connection.rs | 211 ++++++++++++++++++++------------------ src/network/handshake.rs | 8 +- src/network/host.rs | 47 ++++++++- src/network/mod.rs | 18 ++++ src/network/session.rs | 149 +++++++++++++++++++++++++-- src/rlp.rs | 2 +- 6 files changed, 316 insertions(+), 119 deletions(-) diff --git a/src/network/connection.rs b/src/network/connection.rs index db5cde10d..e3a935dde 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -1,4 +1,5 @@ #![allow(dead_code)] //TODO: remove this after everything is done +use std::collections::VecDeque; use mio::{Token, EventSet, EventLoop, Timeout, PollOpt, TryRead, TryWrite}; use mio::tcp::*; use hash::*; @@ -23,15 +24,117 @@ pub struct Connection { pub socket: TcpStream, rec_buf: Bytes, rec_size: usize, - send_buf: Cursor, + send_queue: VecDeque>, interest: EventSet, } +#[derive(PartialEq, Eq)] pub enum WriteStatus { Ongoing, Complete } +impl Connection { + pub fn new(token: Token, socket: TcpStream) -> Connection { + Connection { + token: token, + socket: socket, + send_queue: VecDeque::new(), + rec_buf: Bytes::new(), + rec_size: 0, + interest: EventSet::hup(), + } + } + + pub fn expect(&mut self, size: usize) { + if self.rec_size != self.rec_buf.len() { + warn!(target:"net", "Unexpected connection read start"); + } + unsafe { self.rec_buf.set_len(0) } + self.rec_size = size; + } + + //TODO: return a slice + pub fn readable(&mut self) -> io::Result> { + if self.rec_size == 0 || self.rec_buf.len() >= self.rec_size { + warn!(target:"net", "Unexpected connection read"); + } + let max = self.rec_size - self.rec_buf.len(); + // resolve "multiple applicable items in scope [E0034]" error + let sock_ref = ::by_ref(&mut self.socket); + match sock_ref.take(max as u64).try_read_buf(&mut self.rec_buf) { + Ok(Some(_)) if self.rec_buf.len() == self.rec_size => { + self.rec_size = 0; + Ok(Some(::std::mem::replace(&mut self.rec_buf, Bytes::new()))) + }, + Ok(_) => Ok(None), + Err(e) => Err(e), + } + } + + pub fn send(&mut self, data: Bytes) { //TODO: take ownership version + if data.len() != 0 { + self.send_queue.push_back(Cursor::new(data)); + } + if !self.interest.is_writable() { + self.interest.insert(EventSet::writable()); + } + } + + pub fn writable(&mut self) -> io::Result { + if self.send_queue.is_empty() { + return Ok(WriteStatus::Complete) + } + { + let buf = self.send_queue.front_mut().unwrap(); + let send_size = buf.get_ref().len(); + if (buf.position() as usize) >= send_size { + warn!(target:"net", "Unexpected connection data"); + return Ok(WriteStatus::Complete) + } + match self.socket.try_write_buf(buf) { + Ok(_) if (buf.position() as usize) < send_size => { + self.interest.insert(EventSet::writable()); + Ok(WriteStatus::Ongoing) + }, + Ok(_) if (buf.position() as usize) == send_size => { + self.interest.remove(EventSet::writable()); + Ok(WriteStatus::Complete) + }, + Ok(_) => { panic!("Wrote past buffer");}, + Err(e) => Err(e) + } + }.and_then(|r| if r == WriteStatus::Complete { + self.send_queue.pop_front(); + Ok(r) + } + else { Ok(r) } + ) + } + + pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + trace!(target: "net", "connection register; token={:?}", self.token); + self.interest.insert(EventSet::readable()); + event_loop.register_opt(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + error!("Failed to reregister {:?}, {:?}", self.token, e); + Err(e) + }) + } + + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + trace!(target: "net", "connection reregister; token={:?}", self.token); + event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + error!("Failed to reregister {:?}, {:?}", self.token, e); + Err(e) + }) + } +} + +pub struct Packet { + pub protocol: u16, + pub data: Bytes, +} + enum EncryptedConnectionState { Header, Payload, @@ -99,7 +202,7 @@ impl EncryptedConnection { }) } - pub fn write_packet(&mut self, payload: &[u8]) -> Result<(), Error> { + pub fn send_packet(&mut self, payload: &[u8]) -> Result<(), Error> { let mut header = RlpStream::new(); let len = payload.len() as usize; header.append_raw(&[(len >> 16) as u8, (len >> 8) as u8, len as u8], 1); @@ -120,7 +223,7 @@ impl EncryptedConnection { } self.egress_mac.update(&packet[32..(32 + len + padding)]); self.egress_mac.clone().finalize(&mut packet[(32 + len + padding)..]); - self.connection.send(&packet); + self.connection.send(packet); Ok(()) } @@ -153,7 +256,7 @@ impl EncryptedConnection { Ok(()) } - fn read_payload(&mut self, payload: &[u8]) -> Result { + fn read_payload(&mut self, payload: &[u8]) -> Result { let padding = (16 - (self.payload_len % 16)) % 16; let full_length = (self.payload_len + padding + 16) as usize; if payload.len() != full_length { @@ -170,10 +273,13 @@ impl EncryptedConnection { let mut packet = vec![0u8; self.payload_len as usize]; self.decoder.decrypt(&mut RefReadBuffer::new(&payload[0..(full_length - 16)]), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding"); packet.resize(self.payload_len as usize, 0u8); - Ok(packet) + Ok(Packet { + protocol: self.protocol_id, + data: packet + }) } - pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, Error> { + pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.reregister(event_loop)); match self.read_state { @@ -215,96 +321,3 @@ impl EncryptedConnection { } } -impl Connection { - pub fn new(token: Token, socket: TcpStream) -> Connection { - Connection { - token: token, - socket: socket, - send_buf: Cursor::new(Bytes::new()), - rec_buf: Bytes::new(), - rec_size: 0, - interest: EventSet::hup(), - } - } - - pub fn expect(&mut self, size: usize) { - if self.rec_size != self.rec_buf.len() { - warn!(target:"net", "Unexpected connection read start"); - } - unsafe { self.rec_buf.set_len(0) } - self.rec_size = size; - } - - //TODO: return a slice - pub fn readable(&mut self) -> io::Result> { - if self.rec_size == 0 || self.rec_buf.len() >= self.rec_size { - warn!(target:"net", "Unexpected connection read"); - } - let max = self.rec_size - self.rec_buf.len(); - // resolve "multiple applicable items in scope [E0034]" error - let sock_ref = ::by_ref(&mut self.socket); - match sock_ref.take(max as u64).try_read_buf(&mut self.rec_buf) { - Ok(Some(_)) if self.rec_buf.len() == self.rec_size => { - self.rec_size = 0; - Ok(Some(::std::mem::replace(&mut self.rec_buf, Bytes::new()))) - }, - Ok(_) => Ok(None), - Err(e) => Err(e), - } - } - - pub fn send(&mut self, data: &[u8]) { //TODO: take ownership version - let send_size = self.send_buf.get_ref().len(); - if send_size != 0 || self.send_buf.position() as usize >= send_size { - warn!(target:"net", "Unexpected connection send start"); - } - if self.send_buf.get_ref().capacity() < data.len() { - let capacity = self.send_buf.get_ref().capacity(); - self.send_buf.get_mut().reserve(data.len() - capacity); - } - unsafe { self.send_buf.get_mut().set_len(data.len()) } - unsafe { ::std::ptr::copy_nonoverlapping(data.as_ptr(), self.send_buf.get_mut()[..].as_mut_ptr(), data.len()) }; - if !self.interest.is_writable() { - self.interest.insert(EventSet::writable()); - } - } - - pub fn writable(&mut self) -> io::Result { - let send_size = self.send_buf.get_ref().len(); - if (self.send_buf.position() as usize) >= send_size { - warn!(target:"net", "Unexpected connection data"); - return Ok(WriteStatus::Complete) - } - match self.socket.try_write_buf(&mut self.send_buf) { - Ok(_) if (self.send_buf.position() as usize) < send_size => { - self.interest.insert(EventSet::writable()); - Ok(WriteStatus::Ongoing) - }, - Ok(_) if (self.send_buf.position() as usize) == send_size => { - self.interest.remove(EventSet::writable()); - Ok(WriteStatus::Complete) - }, - Ok(_) => { panic!("Wrote past buffer");}, - Err(e) => Err(e) - } - } - - pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection register; token={:?}", self.token); - self.interest.insert(EventSet::readable()); - event_loop.register_opt(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { - error!("Failed to reregister {:?}, {:?}", self.token, e); - Err(e) - }) - } - - pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection reregister; token={:?}", self.token); - event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { - error!("Failed to reregister {:?}, {:?}", self.token, e); - Err(e) - }) - } -} - - diff --git a/src/network/handshake.rs b/src/network/handshake.rs index 186d6b8a7..7d6c49ae3 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -180,8 +180,8 @@ impl Handshake { self.nonce.copy_to(nonce); } let message = try!(crypto::ecies::encrypt(&self.id, &data)); - self.connection.send(&message[..]); - self.auth_cipher = message; + self.auth_cipher = message.clone(); + self.connection.send(message); self.state = HandshakeState::WritingAuth; Ok(()) } @@ -198,8 +198,8 @@ impl Handshake { self.nonce.copy_to(nonce); } let message = try!(crypto::ecies::encrypt(&self.id, &data)); - self.connection.send(&message[..]); - self.ack_cipher = message; + self.ack_cipher = message.clone(); + self.connection.send(message); self.state = HandshakeState::WritingAck; Ok(()) } diff --git a/src/network/host.rs b/src/network/host.rs index cdb6dd132..f0465bc15 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -12,6 +12,7 @@ use mio::tcp::*; use mio::udp::*; use hash::*; use crypto::*; +use rlp::*; use time::Tm; use network::handshake::Handshake; use network::session::Session; @@ -55,6 +56,7 @@ impl NetworkConfiguration { #[derive(Debug)] struct NodeEndpoint { address: SocketAddr, + address_str: String, udp_port: u16 } @@ -62,6 +64,7 @@ impl NodeEndpoint { fn new(address: SocketAddr) -> NodeEndpoint { NodeEndpoint { address: address, + address_str: address.to_string(), udp_port: address.port() } } @@ -71,6 +74,7 @@ impl NodeEndpoint { match address { Ok(Some(a)) => Ok(NodeEndpoint { address: a, + address_str: s.to_string(), udp_port: a.port() }), Ok(_) => Err(Error::AddressResolve(None)), @@ -178,10 +182,38 @@ pub enum HostMessage { Shutdown } +#[derive(Debug, PartialEq, Eq)] +pub struct CapabilityInfo { + pub protocol: String, + pub version: u32, +} + +impl Encodable for CapabilityInfo { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_list(|e| { + self.protocol.encode(e); + self.version.encode(e); + }); + } +} + +impl Decodable for CapabilityInfo { + fn decode_untrusted(rlp: &UntrustedRlp) -> Result { + Ok(CapabilityInfo { + protocol: try!(String::decode_untrusted(&try!(rlp.at(0)))), + version: try!(u32::decode_untrusted(&try!(rlp.at(1)))), + }) + } +} + pub struct HostInfo { keys: KeyPair, config: NetworkConfiguration, - nonce: H256 + nonce: H256, + pub protocol_version: u32, + pub client_version: String, + pub listen_port: u16, + pub capabilities: Vec } impl HostInfo { @@ -244,12 +276,17 @@ impl Host { let udp_socket = UdpSocket::bound(&addr).unwrap(); event_loop.register_opt(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); + let port = config.listen_address.port(); let mut host = Host { info: HostInfo { keys: KeyPair::create().unwrap(), config: config, - nonce: H256::random() + nonce: H256::random(), + protocol_version: 4, + client_version: "parity".to_string(), + listen_port: port, + capabilities: vec![ CapabilityInfo { protocol: "eth".to_string(), version: 63 }], }, sender: sender, udp_socket: udp_socket, @@ -263,7 +300,6 @@ impl Host { idle_timeout: idle_timeout }; - host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); @@ -429,7 +465,7 @@ impl Host { } fn have_session(&self, id: &NodeId) -> bool { - self.connections.iter().any(|e| match e { &ConnectionEntry::Session(ref s) => s.id.eq(&id), _ => false }) + self.connections.iter().any(|e| match e { &ConnectionEntry::Session(ref s) => s.info.id.eq(&id), _ => false }) } fn connecting_to(&self, id: &NodeId) -> bool { @@ -594,9 +630,10 @@ impl Host { } fn start_session(&mut self, token: Token, event_loop: &mut EventLoop) { + let info = &self.info; self.connections.replace_with(token, |c| { match c { - ConnectionEntry::Handshake(h) => Session::new(h, event_loop) + ConnectionEntry::Handshake(h) => Session::new(h, event_loop, info) .map(|s| Some(ConnectionEntry::Session(s))) .unwrap_or_else(|e| { debug!(target: "net", "Session construction error: {:?}", e); diff --git a/src/network/mod.rs b/src/network/mod.rs index d1f9940c4..6426d523a 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -4,6 +4,23 @@ mod connection; mod handshake; mod session; +#[derive(Debug, Copy, Clone)] +pub enum DisconnectReason +{ + DisconnectRequested, + TCPError, + BadProtocol, + UselessPeer, + TooManyPeers, + DuplicatePeer, + IncompatibleProtocol, + NullIdentity, + ClientQuit, + UnexpectedIdentity, + LocalIdentity, + PingTimeout, +} + #[derive(Debug)] pub enum Error { Crypto(::crypto::CryptoError), @@ -13,6 +30,7 @@ pub enum Error { AddressParse(::std::net::AddrParseError), AddressResolve(Option<::std::io::Error>), NodeIdParse(::error::EthcoreError), + Disconnect(DisconnectReason) } impl From<::std::io::Error> for Error { diff --git a/src/network/session.rs b/src/network/session.rs index 71554b0ca..1ed2449b5 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -1,33 +1,162 @@ #![allow(dead_code)] //TODO: remove this after everything is done -//TODO: remove all unwraps +//TODO: hello packet timeout use mio::*; use hash::*; -use network::connection::{EncryptedConnection}; +use rlp::*; +use network::connection::{EncryptedConnection, Packet}; use network::handshake::Handshake; -use network::Error; +use network::{Error, DisconnectReason}; use network::host::*; pub struct Session { - pub id: NodeId, + pub info: SessionInfo, connection: EncryptedConnection, + had_hello: bool, } +pub struct SessionInfo { + pub id: NodeId, + pub client_version: String, + pub protocol_version: u32, + pub capabilities: Vec, +} + +const PACKET_HELLO: u8 = 0x80; +const PACKET_DISCONNECT: u8 = 0x01; +const PACKET_PING: u8 = 0x02; +const PACKET_PONG: u8 = 0x03; +const PACKET_GET_PEERS: u8 = 0x04; +const PACKET_PEERS: u8 = 0x05; +const PACKET_USER: u8 = 0x10; +const PACKET_LAST: u8 = 0x7f; + impl Session { - pub fn new(h: Handshake, event_loop: &mut EventLoop) -> Result { + pub fn new(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { let id = h.id.clone(); let mut connection = try!(EncryptedConnection::new(h)); try!(connection.register(event_loop)); - Ok(Session { - id: id, + let mut session = Session { connection: connection, - }) + had_hello: false, + info: SessionInfo { + id: id, + client_version: String::new(), + protocol_version: 0, + capabilities: Vec::new(), + }, + }; + try!(session.write_hello(host)); + try!(session.write_ping()); + Ok(session) } - pub fn readable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { - try!(self.connection.readable(event_loop)); + + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { + match try!(self.connection.readable(event_loop)) { + Some(data) => { + try!(self.read_packet(data, host)); + }, + None => {} + }; Ok(()) } + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { self.connection.writable(event_loop) } + + pub fn read_packet(&mut self, packet: Packet, host: &HostInfo) -> Result<(), Error> { + let data = &packet.data; + if data.len() < 2 { + return Err(Error::BadProtocol); + } + let packet_id = data[0]; + let rlp = UntrustedRlp::new(&data[1..]); //TODO: validate rlp expected size + if packet_id != PACKET_HELLO && packet_id != PACKET_DISCONNECT && !self.had_hello { + return Err(Error::BadProtocol); + } + match packet_id { + PACKET_HELLO => self.read_hello(&rlp, host), + PACKET_DISCONNECT => Err(Error::Disconnect(DisconnectReason::DisconnectRequested)), + PACKET_PING => self.write_pong(), + PACKET_GET_PEERS => Ok(()), //TODO; + PACKET_PEERS => Ok(()), + PACKET_USER ... PACKET_LAST => { + warn!(target: "net", "User packet: {:?}", rlp); + Ok(()) + }, + _ => { + debug!(target: "net", "Unkown packet: {:?}", rlp); + Ok(()) + } + } + } + + fn write_hello(&mut self, host: &HostInfo) -> Result<(), Error> { + let mut rlp = RlpStream::new(); + rlp.append(&(PACKET_HELLO as u32)); + rlp.append_list(5) + .append(&host.protocol_version) + .append(&host.client_version) + .append(&host.capabilities) + .append(&host.listen_port) + .append(host.id()); + self.connection.send_packet(&rlp.out()) + } + + fn read_hello(&mut self, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), Error> { + let protocol = try!(u32::decode_untrusted(&try!(rlp.at(0)))); + let client_version = try!(String::decode_untrusted(&try!(rlp.at(0)))); + let mut caps: Vec = try!(Decodable::decode_untrusted(&try!(rlp.at(2)))); + let id = try!(NodeId::decode_untrusted(&try!(rlp.at(4)))); + + // Intersect with host capabilities + // Leave only highset mutually supported capability version + caps.retain(|c| host.capabilities.contains(&c)); + let mut i = 0; + while i < caps.len() { + if caps.iter().any(|c| c.protocol == caps[i].protocol && c.version > caps[i].version) { + caps.remove(i); + } + else { + i += 1; + } + } + + trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); + if protocol != host.protocol_version { + return Err(self.disconnect(DisconnectReason::UselessPeer)); + } + self.had_hello = true; + Ok(()) + } + + fn write_ping(&mut self) -> Result<(), Error> { + self.send(try!(Session::prepare(PACKET_PING, 0))) + } + + fn write_pong(&mut self) -> Result<(), Error> { + self.send(try!(Session::prepare(PACKET_PONG, 0))) + } + + + fn disconnect(&mut self, reason: DisconnectReason) -> Error { + let mut rlp = RlpStream::new(); + rlp.append(&(PACKET_DISCONNECT as u32)); + rlp.append_list(1); + rlp.append(&(reason.clone() as u32)); + self.connection.send_packet(&rlp.out()).ok(); + Error::Disconnect(reason) + } + + fn prepare(packet_id: u8, items: usize) -> Result { + let mut rlp = RlpStream::new_list(1); + rlp.append(&(packet_id as u32)); + rlp.append_list(items); + Ok(rlp) + } + + fn send(&mut self, rlp: RlpStream) -> Result<(), Error> { + self.connection.send_packet(&rlp.out()) + } } diff --git a/src/rlp.rs b/src/rlp.rs index c023b1dfb..cb3fa378c 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -725,7 +725,7 @@ pub trait Decoder { fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result; } -struct BasicDecoder; +pub struct BasicDecoder; impl Decoder for BasicDecoder { fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { From 2cbbc872ff14f7462f0f06ff01374587b1773a9c Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 21:49:57 +0100 Subject: [PATCH 195/379] json-tests init --- Cargo.toml | 3 + json-tests/Cargo.toml | 8 +++ json-tests/README.md | 15 +++++ json-tests/json/trie/README.md | 35 ++++++++++ json-tests/json/trie/basic.json | 11 ++++ json-tests/src/lib.rs | 68 +++++++++++++++++++ json-tests/src/trie.rs | 113 ++++++++++++++++++++++++++++++++ src/triehash.rs | 85 +++--------------------- 8 files changed, 263 insertions(+), 75 deletions(-) create mode 100644 json-tests/Cargo.toml create mode 100644 json-tests/README.md create mode 100644 json-tests/json/trie/README.md create mode 100644 json-tests/json/trie/basic.json create mode 100644 json-tests/src/lib.rs create mode 100644 json-tests/src/trie.rs diff --git a/Cargo.toml b/Cargo.toml index de4dcd41b..41ca44580 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,6 @@ num = "0.1" lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" + +[dev-dependencies] +json-tests = { path = "json-tests" } diff --git a/json-tests/Cargo.toml b/json-tests/Cargo.toml new file mode 100644 index 000000000..3185e2e59 --- /dev/null +++ b/json-tests/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "json-tests" +version = "0.1.0" +authors = ["debris "] + +[dependencies] +rustc-serialize = "0.3" +glob = "*" diff --git a/json-tests/README.md b/json-tests/README.md new file mode 100644 index 000000000..170cd588d --- /dev/null +++ b/json-tests/README.md @@ -0,0 +1,15 @@ +# How to write json test file? + +Cause it's very hard to write generic json test files, each subdirectory should follow its own +convention. BUT all json files `within` same directory should be consistent. + +### Test file should always contain a single file with input and output. + +```json +{ + input: ..., + output: ... +} +``` + +As a reference, please use trietests. diff --git a/json-tests/json/trie/README.md b/json-tests/json/trie/README.md new file mode 100644 index 000000000..6d7f479a3 --- /dev/null +++ b/json-tests/json/trie/README.md @@ -0,0 +1,35 @@ +# Trie tests guideline + +Trie test input is an array of operations. Each operation must have 2 fields: + +- `operation` - string, either `insert` or `remove` +- `key` - string, or hex value prefixed with `0x` + +And optional field: + +- `value`- which is used by `insert` operation + +### Example + +```json +{ + "input": + [ + { + "operation": "insert", + "key": "world", + "value": "hello" + }, + { + "operation": "insert", + "key": "0x1234", + "value": "ooooops" + }, + { + "operation": "remove", + "key": "0x1234" + } + ], + "output": "0x5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84" +} +``` diff --git a/json-tests/json/trie/basic.json b/json-tests/json/trie/basic.json new file mode 100644 index 000000000..f737ef337 --- /dev/null +++ b/json-tests/json/trie/basic.json @@ -0,0 +1,11 @@ +{ + "input": + [ + { + "operation": "insert", + "key": "A", + "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ], + "output": "0xd23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab" +} diff --git a/json-tests/src/lib.rs b/json-tests/src/lib.rs new file mode 100644 index 000000000..64466084a --- /dev/null +++ b/json-tests/src/lib.rs @@ -0,0 +1,68 @@ +extern crate rustc_serialize; +extern crate glob; + +use std::str::from_utf8; +use std::path::*; +use std::io::prelude::*; +use std::fs::File; +use glob::glob; +use rustc_serialize::*; + +pub mod trie; + +pub trait JsonTest: Sized { + type Input; + type Output; + + fn new(data: &[u8]) -> Self; + fn input(&self) -> Self::Input; + fn output(&self) -> Self::Output; +} + +pub struct JsonLoader { + json: json::Json +} + +impl JsonTest for JsonLoader { + type Input = json::Json; + type Output = json::Json; + + fn new(data: &[u8]) -> Self { + JsonLoader { + json: json::Json::from_str(from_utf8(data).unwrap()).unwrap() + } + } + fn input(&self) -> Self::Input { + self.json.as_object().unwrap()["input"].clone() + } + + fn output(&self) -> Self::Output { + self.json.as_object().unwrap()["output"].clone() + } +} + +pub fn execute_test(data: &[u8], f: &mut F) where T: JsonTest, F: FnMut(T::Input, T::Output) { + let test = T::new(data); + f(test.input(), test.output()) +} + +pub fn execute_test_from_file(path: &Path, f: &mut F) where T: JsonTest, F: FnMut(T::Input, T::Output) { + let mut file = File::open(path).unwrap(); + let mut buffer = vec![]; + let _ = file.read_to_end(&mut buffer); + let test = T::new(&buffer); + f(test.input(), test.output()) +} + +pub fn execute_tests_in_directory(pattern: &str, f: &mut F) where T: JsonTest, F: FnMut(String, T::Input, T::Output) { + for path in glob(pattern).unwrap().filter_map(Result::ok) { + execute_test_from_file::(&path, &mut | input, output | { + f(path.to_str().unwrap().to_string(), input, output); + }); + } +} + +#[test] +fn it_works() { +} + diff --git a/json-tests/src/trie.rs b/json-tests/src/trie.rs new file mode 100644 index 000000000..f065085fe --- /dev/null +++ b/json-tests/src/trie.rs @@ -0,0 +1,113 @@ +//! json trie tests +use std::collections::HashMap; +use rustc_serialize::*; +use rustc_serialize::hex::FromHex; +use super::{JsonTest, JsonLoader}; + +pub enum OperationType { + Insert, + Remove +} + +impl Decodable for OperationType { + fn decode(d: &mut D) -> Result where D: Decoder { + match try!(String::decode(d)).as_ref() { + "insert" => Ok(OperationType::Insert), + "remove" => Ok(OperationType::Remove), + other => panic!("invalid operation type: {}", other) + } + } +} + +#[derive(RustcDecodable)] +struct RawOperation { + operation: OperationType, + key: String, + value: Option +} + +pub struct Operation { + pub operation: OperationType, + pub key: Vec, + pub value: Option> +} + +fn hex_or_string(s: &str) -> Vec { + match s.starts_with("0x") { + true => s[2..].from_hex().unwrap(), + false => From::from(s) + } +} + +impl Into for RawOperation { + fn into(self) -> Operation { + Operation { + operation: self.operation, + key: hex_or_string(&self.key), + value: self.value.map(|v| { + hex_or_string(&v) + }) + } + } +} + +pub struct TrieTest { + loader: JsonLoader +} + +impl JsonTest for TrieTest { + type Input = Vec; + type Output = Vec; + + fn new(data: &[u8]) -> Self { + TrieTest { + loader: JsonLoader::new(data) + } + } + + fn input(&self) -> Self::Input { + let mut decoder = json::Decoder::new(self.loader.input()); + let raw: Vec = Decodable::decode(&mut decoder).unwrap(); + raw.into_iter() + .map(|i| i.into()) + .collect() + } + fn output(&self) -> Self::Output { + hex_or_string(self.loader.output().as_string().unwrap()) + } +} + +pub struct TriehashTest { + trietest: TrieTest +} + +impl JsonTest for TriehashTest { + type Input = Vec<(Vec, Vec)>; + type Output = Vec; + + fn new(data: &[u8]) -> Self { + TriehashTest { + trietest: TrieTest::new(data) + } + } + + fn input(&self) -> Self::Input { + self.trietest.input() + .into_iter() + .fold(HashMap::new(), | mut map, o | { + match o.operation { + OperationType::Insert => map.insert(o.key, o.value.unwrap()), + OperationType::Remove => map.remove(&o.key) + }; + map + }) + .into_iter() + .map(|p| { p }) + .collect() + } + + fn output(&self) -> Self::Output { + self.trietest.output() + } +} + diff --git a/src/triehash.rs b/src/triehash.rs index ab0c75683..0467ea284 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -379,80 +379,15 @@ mod tests { assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); } - //fn hex_or_string(s: &str) -> Vec { - //match s.starts_with("0x") { - //true => s[2..].from_hex().unwrap(), - //false => From::from(s) - //} - //} - - //pub fn yield_json_trietest(data: &[u8], name: &str, insert: &mut I, remove: &mut R) -> H256 where I: FnMut(Vec, Vec), R: FnMut(Vec) { - ////let data = include_bytes!("../tests/TrieTests/trietest.json"); - - //let s = String::from_bytes(data).unwrap(); - //let json = Json::from_str(&s).unwrap(); - //let obj = json.as_object().unwrap(); - //println!("here"); - - //let value = &obj[name]; - - //let i = &value["in"]; - //let o = &value["root"]; - - //let root_str = o.as_string().unwrap(); - - //println!("here2"); - //for i in i.as_array().unwrap().iter() { - //let key = hex_or_string(i[0].as_string().unwrap()); - - //match i[1].is_null() { - //true => remove(key), - //false => insert(key, hex_or_string(i[1].as_string().unwrap())) - //}; - //} - - //H256::from_str(&root_str[2..]).unwrap() - //} - - ////fn load_json_trietest(data: &[u8], name: &str) -> (Vec<(Vec, Vec)>, H256) { - ////use std::cell::RefCell; - //let map = RefCell::new(HashMap::new()); - //let root = yield_json_trietest(data, name, &mut | key, value | { - //map.borrow_mut().insert(key, value); - //}, &mut | key | { - //map.borrow_mut().remove(&key); - //}); - - //let res = map.into_inner() - //.into_iter() - //.map(|p| p) - //.collect(); - //(res, root) - //} - - //#[test] - //fn triehash_json_empty_values() { - //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/trietest.json"), "emptyValues"); - //assert_eq!(trie_root(input), root); - //} - - //#[test] - //fn triehash_json_branching_tests() { - //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/trietest.json"), "branchingTests"); - //assert_eq!(trie_root(input), root); - //} - - //#[test] - //fn triehash_json_jeff_tests() { - //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/trietest.json"), "jeff"); - //assert_eq!(trie_root(input), root); - //} - - ////#[test] - ////fn triehash_json_test1() { - //let (input, root) = load_json_trietest(include_bytes!("../tests/TrieTests/hex_encoded_securetrie_test.json"), "test1"); - //assert_eq!(trie_root(input), root); - //} - + extern crate json_tests; + use self::json_tests::*; + #[test] + fn run_trie_tests() { + execute_tests_in_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { + println!("file: {}, output: {:?}", file, output); + assert_eq!(trie_root(input), H256::from_slice(&output)); + }); + } } + From 51bd813734038726cfa04cb888ee91b070400b25 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 21:51:20 +0100 Subject: [PATCH 196/379] updated gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index eabd0a44e..959045cf9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ Cargo.lock # Generated by Cargo /target/ +/json-tests/target/ # Vim *.swp From b9181b42bc7dfb8673751e3eb8e84a2fa3ce2156 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 22:49:47 +0100 Subject: [PATCH 197/379] *.json test files --- json-tests/json/trie/branching.json | 229 +++++++++++++++++++++++++ json-tests/json/trie/dogs.json | 21 +++ json-tests/json/trie/empty.json | 4 + json-tests/json/trie/empty_values.json | 44 +++++ json-tests/json/trie/foo.json | 16 ++ json-tests/json/trie/jeff.json | 58 +++++++ json-tests/src/trie.rs | 1 + src/triehash.rs | 34 ---- 8 files changed, 373 insertions(+), 34 deletions(-) create mode 100644 json-tests/json/trie/branching.json create mode 100644 json-tests/json/trie/dogs.json create mode 100644 json-tests/json/trie/empty.json create mode 100644 json-tests/json/trie/empty_values.json create mode 100644 json-tests/json/trie/foo.json create mode 100644 json-tests/json/trie/jeff.json diff --git a/json-tests/json/trie/branching.json b/json-tests/json/trie/branching.json new file mode 100644 index 000000000..7bbb13f03 --- /dev/null +++ b/json-tests/json/trie/branching.json @@ -0,0 +1,229 @@ +{ + "input": [ + { + "operation": "insert", + "key": "0x04110d816c380812a427968ece99b1c963dfbce6", + "value": "something" + }, + { + "operation": "insert", + "key": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": "something" + }, + { + "operation": "insert", + "key": "0x0a517d755cebbf66312b30fff713666a9cb917e0", + "value": "something" + }, + { + "operation": "insert", + "key": "0x24dd378f51adc67a50e339e8031fe9bd4aafab36", + "value": "something" + }, + { + "operation": "insert", + "key": "0x293f982d000532a7861ab122bdc4bbfd26bf9030", + "value": "something" + }, + { + "operation": "insert", + "key": "0x2cf5732f017b0cf1b1f13a1478e10239716bf6b5", + "value": "something" + }, + { + "operation": "insert", + "key": "0x31c640b92c21a1f1465c91070b4b3b4d6854195f", + "value": "something" + }, + { + "operation": "insert", + "key": "0x37f998764813b136ddf5a754f34063fd03065e36", + "value": "something" + }, + { + "operation": "insert", + "key": "0x37fa399a749c121f8a15ce77e3d9f9bec8020d7a", + "value": "something" + }, + { + "operation": "insert", + "key": "0x4f36659fa632310b6ec438dea4085b522a2dd077", + "value": "something" + }, + { + "operation": "insert", + "key": "0x62c01474f089b07dae603491675dc5b5748f7049", + "value": "something" + }, + { + "operation": "insert", + "key": "0x729af7294be595a0efd7d891c9e51f89c07950c7", + "value": "something" + }, + { + "operation": "insert", + "key": "0x83e3e5a16d3b696a0314b30b2534804dd5e11197", + "value": "something" + }, + { + "operation": "insert", + "key": "0x8703df2417e0d7c59d063caa9583cb10a4d20532", + "value": "something" + }, + { + "operation": "insert", + "key": "0x8dffcd74e5b5923512916c6a64b502689cfa65e1", + "value": "something" + }, + { + "operation": "insert", + "key": "0x95a4d7cccb5204733874fa87285a176fe1e9e240", + "value": "something" + }, + { + "operation": "insert", + "key": "0x99b2fcba8120bedd048fe79f5262a6690ed38c39", + "value": "something" + }, + { + "operation": "insert", + "key": "0xa4202b8b8afd5354e3e40a219bdc17f6001bf2cf", + "value": "something" + }, + { + "operation": "insert", + "key": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "value": "something" + }, + { + "operation": "insert", + "key": "0xa9647f4a0a14042d91dc33c0328030a7157c93ae", + "value": "something" + }, + { + "operation": "insert", + "key": "0xaa6cffe5185732689c18f37a7f86170cb7304c2a", + "value": "something" + }, + { + "operation": "insert", + "key": "0xaae4a2e3c51c04606dcb3723456e58f3ed214f45", + "value": "something" + }, + { + "operation": "insert", + "key": "0xc37a43e940dfb5baf581a0b82b351d48305fc885", + "value": "something" + }, + { + "operation": "insert", + "key": "0xd2571607e241ecf590ed94b12d87c94babe36db6", + "value": "something" + }, + { + "operation": "insert", + "key": "0xf735071cbee190d76b704ce68384fc21e389fbe7", + "value": "something" + }, + { + "operation": "remove", + "key": "0x04110d816c380812a427968ece99b1c963dfbce6" + }, + { + "operation": "remove", + "key": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87" + }, + { + "operation": "remove", + "key": "0x0a517d755cebbf66312b30fff713666a9cb917e0" + }, + { + "operation": "remove", + "key": "0x24dd378f51adc67a50e339e8031fe9bd4aafab36" + }, + { + "operation": "remove", + "key": "0x293f982d000532a7861ab122bdc4bbfd26bf9030" + }, + { + "operation": "remove", + "key": "0x2cf5732f017b0cf1b1f13a1478e10239716bf6b5" + }, + { + "operation": "remove", + "key": "0x31c640b92c21a1f1465c91070b4b3b4d6854195f" + }, + { + "operation": "remove", + "key": "0x37f998764813b136ddf5a754f34063fd03065e36" + }, + { + "operation": "remove", + "key": "0x37fa399a749c121f8a15ce77e3d9f9bec8020d7a" + }, + { + "operation": "remove", + "key": "0x4f36659fa632310b6ec438dea4085b522a2dd077" + }, + { + "operation": "remove", + "key": "0x62c01474f089b07dae603491675dc5b5748f7049" + }, + { + "operation": "remove", + "key": "0x729af7294be595a0efd7d891c9e51f89c07950c7" + }, + { + "operation": "remove", + "key": "0x83e3e5a16d3b696a0314b30b2534804dd5e11197" + }, + { + "operation": "remove", + "key": "0x8703df2417e0d7c59d063caa9583cb10a4d20532" + }, + { + "operation": "remove", + "key": "0x8dffcd74e5b5923512916c6a64b502689cfa65e1" + }, + { + "operation": "remove", + "key": "0x95a4d7cccb5204733874fa87285a176fe1e9e240" + }, + { + "operation": "remove", + "key": "0x99b2fcba8120bedd048fe79f5262a6690ed38c39" + }, + { + "operation": "remove", + "key": "0xa4202b8b8afd5354e3e40a219bdc17f6001bf2cf" + }, + { + "operation": "remove", + "key": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + }, + { + "operation": "remove", + "key": "0xa9647f4a0a14042d91dc33c0328030a7157c93ae" + }, + { + "operation": "remove", + "key": "0xaa6cffe5185732689c18f37a7f86170cb7304c2a" + }, + { + "operation": "remove", + "key": "0xaae4a2e3c51c04606dcb3723456e58f3ed214f45" + }, + { + "operation": "remove", + "key": "0xc37a43e940dfb5baf581a0b82b351d48305fc885" + }, + { + "operation": "remove", + "key": "0xd2571607e241ecf590ed94b12d87c94babe36db6" + }, + { + "operation": "remove", + "key": "0xf735071cbee190d76b704ce68384fc21e389fbe7" + }], + "output": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" +} \ No newline at end of file diff --git a/json-tests/json/trie/dogs.json b/json-tests/json/trie/dogs.json new file mode 100644 index 000000000..a6ff7c891 --- /dev/null +++ b/json-tests/json/trie/dogs.json @@ -0,0 +1,21 @@ +{ + "input": + [ + { + "operation": "insert", + "key": "doe", + "value": "reindeer" + }, + { + "operation": "insert", + "key": "dogglesworth", + "value": "cat" + }, + { + "operation": "insert", + "key": "dog", + "value": "puppy" + } + ], + "output": "0x8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3" +} diff --git a/json-tests/json/trie/empty.json b/json-tests/json/trie/empty.json new file mode 100644 index 000000000..ca146df54 --- /dev/null +++ b/json-tests/json/trie/empty.json @@ -0,0 +1,4 @@ +{ + "input": [], + "output": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" +} diff --git a/json-tests/json/trie/empty_values.json b/json-tests/json/trie/empty_values.json new file mode 100644 index 000000000..ac367f913 --- /dev/null +++ b/json-tests/json/trie/empty_values.json @@ -0,0 +1,44 @@ +{ + "input": + [ + { + "operation": "insert", + "key": "do", + "value": "verb" + }, + { + "operation": "insert", + "key": "ether", + "value": "wookiedoo" + }, + { + "operation": "insert", + "key": "horse", + "value": "stallion" + }, + { + "operation": "insert", + "key": "shaman", + "value": "horse" + }, + { + "operation": "insert", + "key": "doge", + "value": "coin" + }, + { + "operation": "remove", + "key": "ether" + }, + { + "operation": "insert", + "key": "dog", + "value": "puppy" + }, + { + "operation": "remove", + "key": "shaman" + } + ], + "output": "0x5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84" +} diff --git a/json-tests/json/trie/foo.json b/json-tests/json/trie/foo.json new file mode 100644 index 000000000..4b8c0a87f --- /dev/null +++ b/json-tests/json/trie/foo.json @@ -0,0 +1,16 @@ +{ + "input": + [ + { + "operation": "insert", + "key": "foo", + "value": "bar" + }, + { + "operation": "insert", + "key": "food", + "value": "bass" + } + ], + "output": "0x17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3" +} diff --git a/json-tests/json/trie/jeff.json b/json-tests/json/trie/jeff.json new file mode 100644 index 000000000..1f3093fad --- /dev/null +++ b/json-tests/json/trie/jeff.json @@ -0,0 +1,58 @@ +{ + "input": [ + { + "operation": "insert", + "key": "0x0000000000000000000000000000000000000000000000000000000000000045", + "value": "0x22b224a1420a802ab51d326e29fa98e34c4f24ea" + }, + { + "operation": "insert", + "key": "0x0000000000000000000000000000000000000000000000000000000000000046", + "value": "0x67706c2076330000000000000000000000000000000000000000000000000000" + }, + { + "operation": "insert", + "key": "0x0000000000000000000000000000000000000000000000000000001234567890", + "value": "0x697c7b8c961b56f675d570498424ac8de1a918f6" + }, + { + "operation": "insert", + "key": "0x000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6", + "value": "0x1234567890" + }, + { + "operation": "insert", + "key": "0x0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2", + "value": "0x4655474156000000000000000000000000000000000000000000000000000000" + }, + { + "operation": "insert", + "key": "0x000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1", + "value": "0x4e616d6552656700000000000000000000000000000000000000000000000000" + }, + { + "operation": "insert", + "key": "0x4655474156000000000000000000000000000000000000000000000000000000", + "value": "0x7ef9e639e2733cb34e4dfc576d4b23f72db776b2" + }, + { + "operation": "insert", + "key": "0x4e616d6552656700000000000000000000000000000000000000000000000000", + "value": "0xec4f34c97e43fbb2816cfd95e388353c7181dab1" + }, + { + "operation": "remove", + "key": "0x0000000000000000000000000000000000000000000000000000001234567890" + }, + { + "operation": "insert", + "key": "0x000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6", + "value": "0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000" + }, + { + "operation": "insert", + "key": "0x6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000", + "value": "0x697c7b8c961b56f675d570498424ac8de1a918f6" + }], + "output": "0x9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100" +} \ No newline at end of file diff --git a/json-tests/src/trie.rs b/json-tests/src/trie.rs index f065085fe..f6d4eaae1 100644 --- a/json-tests/src/trie.rs +++ b/json-tests/src/trie.rs @@ -72,6 +72,7 @@ impl JsonTest for TrieTest { .map(|i| i.into()) .collect() } + fn output(&self) -> Self::Output { hex_or_string(self.loader.output().as_string().unwrap()) } diff --git a/src/triehash.rs b/src/triehash.rs index 0467ea284..a8b1670d3 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -285,40 +285,6 @@ mod tests { use hash::*; use triehash::*; - #[test] - fn empty_trie_root() { - assert_eq!(trie_root(vec![]), H256::from_str("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap()); - } - - #[test] - fn single_trie_item() { - let v = vec![(From::from("A"), From::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))]; - assert_eq!(trie_root(v), H256::from_str("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab").unwrap()); - } - - #[test] - fn foo_trie_item() { - - let v = vec![ - (From::from("foo"), From::from("bar")), - (From::from("food"), From::from("bass")) - ]; - - assert_eq!(trie_root(v), H256::from_str("17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3").unwrap()); - } - - #[test] - fn dogs_trie_item() { - - let v = vec![ - (From::from("doe"), From::from("reindeer")), - (From::from("dog"), From::from("puppy")), - (From::from("dogglesworth"), From::from("cat")), - ]; - - assert_eq!(trie_root(v), H256::from_str("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3").unwrap()); - } - #[test] fn puppy_trie_items() { From 166b84d489ffe2ddfe2d282be9c096db4c8553e8 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 22:56:38 +0100 Subject: [PATCH 198/379] removed duplicate tests --- json-tests/src/lib.rs | 6 +---- src/triehash.rs | 62 ++++--------------------------------------- 2 files changed, 6 insertions(+), 62 deletions(-) diff --git a/json-tests/src/lib.rs b/json-tests/src/lib.rs index 64466084a..8a800e8f9 100644 --- a/json-tests/src/lib.rs +++ b/json-tests/src/lib.rs @@ -54,7 +54,7 @@ pub fn execute_test_from_file(path: &Path, f: &mut F) where T: JsonTest, F f(test.input(), test.output()) } -pub fn execute_tests_in_directory(pattern: &str, f: &mut F) where T: JsonTest, F: FnMut(String, T::Input, T::Output) { +pub fn execute_tests_from_directory(pattern: &str, f: &mut F) where T: JsonTest, F: FnMut(String, T::Input, T::Output) { for path in glob(pattern).unwrap().filter_map(Result::ok) { execute_test_from_file::(&path, &mut | input, output | { f(path.to_str().unwrap().to_string(), input, output); @@ -62,7 +62,3 @@ pub fn execute_tests_in_directory(pattern: &str, f: &mut F) where T: JsonT } } -#[test] -fn it_works() { -} - diff --git a/src/triehash.rs b/src/triehash.rs index a8b1670d3..6cfa2103d 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -277,29 +277,13 @@ fn test_hex_prefix_encode() { #[cfg(test)] mod tests { - use std::str::FromStr; - use std::collections::HashMap; - use rustc_serialize::hex::FromHex; - use rustc_serialize::json::Json; - use bytes::*; + extern crate json_tests; + use self::json_tests::*; use hash::*; use triehash::*; #[test] - fn puppy_trie_items() { - - let v = vec![ - (From::from("do"), From::from("verb")), - (From::from("dog"), From::from("puppy")), - (From::from("doge"), From::from("coin")), - (From::from("horse"), From::from("stallion")), - ]; - - assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); - } - - #[test] - fn out_of_order() { + fn test_trie_out_of_order() { assert!(trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), @@ -313,44 +297,8 @@ mod tests { } #[test] - fn test_trie_root() { - let v = vec![ - - ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), - "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), - - ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), - "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - - ("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), - "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), - - ("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), - "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - - ("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), - "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), - - ("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), - "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), - - ("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), - "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), - - ("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), - "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) - - ]; - - assert_eq!(trie_root(v), H256::from_str("9f6221ebb8efe7cff60a716ecb886e67dd042014be444669f0159d8e68b42100").unwrap()); - } - - extern crate json_tests; - use self::json_tests::*; - - #[test] - fn run_trie_tests() { - execute_tests_in_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { + fn test_trie_json() { + execute_tests_from_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { println!("file: {}, output: {:?}", file, output); assert_eq!(trie_root(input), H256::from_slice(&output)); }); From 8b481d5e11bcdf4c6907da7f02ae7a10d310bf12 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 22:59:00 +0100 Subject: [PATCH 199/379] Remove. Untested. --- src/nibbleslice.rs | 45 +++- src/trie.rs | 607 ++++++++++++++++++++++++++++++--------------- src/triehash.rs | 4 +- 3 files changed, 443 insertions(+), 213 deletions(-) diff --git a/src/nibbleslice.rs b/src/nibbleslice.rs index 5f2615991..6f4232945 100644 --- a/src/nibbleslice.rs +++ b/src/nibbleslice.rs @@ -30,6 +30,8 @@ use bytes::*; pub struct NibbleSlice<'a> { data: &'a [u8], offset: usize, + data_encode_suffix: &'a [u8], + offset_encode_suffix: usize, } impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { @@ -37,7 +39,26 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { pub fn new(data: &[u8]) -> NibbleSlice { NibbleSlice::new_offset(data, 0) } /// Create a new nibble slice with the given byte-slice with a nibble offset. - pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset} } + pub fn new_offset(data: &'a [u8], offset: usize) -> NibbleSlice { NibbleSlice{data: data, offset: offset, data_encode_suffix: &b""[..], offset_encode_suffix: 0} } + + /// + pub fn new_composed(a: &'a NibbleSlice, b: &'a NibbleSlice) -> NibbleSlice<'a> { NibbleSlice{data: a.data, offset: a.offset, data_encode_suffix: b.data, offset_encode_suffix: b.offset} } + + /*pub fn new_composed_bytes_offset(a: &NibbleSlice, b: &NibbleSlice) -> (Bytes, usize) { + let r: Vec::with_capacity((a.len() + b.len() + 1) / 2); + let mut i = (a.len() + b.len()) % 2; + while i < a.len() { + match i % 2 { + 0 => , + 1 => , + } + i += 1; + } + while i < a.len() + b.len() { + i += 1; + } + (r, a.len() + b.len()) + }*/ /// Create a new nibble slice from the given HPE encoded data (e.g. output of `encoded()`). pub fn from_encoded(data: &'a [u8]) -> (NibbleSlice, bool) { @@ -48,20 +69,32 @@ impl<'a, 'view> NibbleSlice<'a> where 'a: 'view { pub fn is_empty(&self) -> bool { self.len() == 0 } /// Get the length (in nibbles, naturally) of this slice. - pub fn len(&self) -> usize { self.data.len() * 2 - self.offset } + pub fn len(&self) -> usize { (self.data.len() + self.data_encode_suffix.len()) * 2 - self.offset - self.offset_encode_suffix } /// Get the nibble at position `i`. pub fn at(&self, i: usize) -> u8 { - if (self.offset + i) & 1 == 1 { - self.data[(self.offset + i) / 2] & 15u8 + let l = self.data.len() * 2 - self.offset; + if i < l { + if (self.offset + i) & 1 == 1 { + self.data[(self.offset + i) / 2] & 15u8 + } + else { + self.data[(self.offset + i) / 2] >> 4 + } } else { - self.data[(self.offset + i) / 2] >> 4 + let i = i - l; + if (self.offset_encode_suffix + i) & 1 == 1 { + self.data_encode_suffix[(self.offset_encode_suffix + i) / 2] & 15u8 + } + else { + self.data_encode_suffix[(self.offset_encode_suffix + i) / 2] >> 4 + } } } /// Return object which represents a view on to this slice (further) offset by `i` nibbles. - pub fn mid(&'view self, i: usize) -> NibbleSlice<'a> { NibbleSlice{ data: self.data, offset: self.offset + i} } + pub fn mid(&'view self, i: usize) -> NibbleSlice<'a> { NibbleSlice{ data: self.data, offset: self.offset + i, data_encode_suffix: &b""[..], offset_encode_suffix: 0 } } /// Do we start with the same nibbles as the whole of `them`? pub fn starts_with(&self, them: &Self) -> bool { self.common_prefix(them) == them.len() } diff --git a/src/trie.rs b/src/trie.rs index 704db94d4..b205eb842 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -33,74 +33,6 @@ pub enum Node<'a> { Branch([&'a[u8]; 16], Option<&'a [u8]>) } -impl <'a>Node<'a> { - pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> { - let r = Rlp::new(node_rlp); - match r.prototype() { - // either leaf or extension - decode first item with NibbleSlice::??? - // and use is_leaf return to figure out which. - // if leaf, second item is a value (is_data()) - // if extension, second item is a node (either SHA3 to be looked up and - // fed back into this function or inline RLP which can be fed back into this function). - Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { - (slice, true) => Node::Leaf(slice, r.at(1).data()), - (slice, false) => Node::Extension(slice, r.at(1).raw()), - }, - // branch - first 16 are nodes, 17th is a value (or empty). - Prototype::List(17) => { - let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; - for i in 0..16 { - nodes[i] = r.at(i).raw(); - } - Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) - }, - // an empty branch index. - Prototype::Data(0) => Node::Empty, - // something went wrong. - _ => panic!("Rlp is not valid.") - } - } - - // todo: should check length before encoding, cause it may just be sha3 of data - pub fn encoded(&self) -> Bytes { - match *self { - Node::Leaf(ref slice, ref value) => { - let mut stream = RlpStream::new_list(2); - stream.append(&slice.encoded(true)); - stream.append(value); - stream.out() - }, - Node::Extension(ref slice, ref raw_rlp) => { - let mut stream = RlpStream::new_list(2); - stream.append(&slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - stream.out() - }, - Node::Branch(ref nodes, ref value) => { - let mut stream = RlpStream::new_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(n) => { stream.append(&n); }, - None => { stream.append_empty_data(); }, - } - stream.out() - }, - Node::Empty => { - let mut stream = RlpStream::new(); - stream.append_empty_data(); - stream.out() - } - } - } -} - -//enum ValidationResult<'a> { - //Valid, - //Invalid { node: Node<'a>, depth: usize } -//} - enum Operation { New(H256, Bytes), Delete(H256), @@ -137,12 +69,125 @@ impl Diff { } } + fn delete_node_from_slice(&mut self, old: &[u8]) { + let r = Rlp::new(old); + if r.is_data() && r.size() == 32 { + self.0.push(Operation::Delete(H256::decode(&r))); + } + } + fn replace_node(&mut self, old: &Rlp, rlp: Bytes, out: &mut RlpStream) { self.delete_node(old); self.new_node(rlp, out); } } +impl <'a>Node<'a> { + fn decoded(node_rlp: &'a [u8]) -> Node<'a> { + let r = Rlp::new(node_rlp); + match r.prototype() { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { + (slice, true) => Node::Leaf(slice, r.at(1).data()), + (slice, false) => Node::Extension(slice, r.at(1).raw()), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { + nodes[i] = r.at(i).raw(); + } + Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) + }, + // an empty branch index. + Prototype::Data(0) => Node::Empty, + // something went wrong. + _ => panic!("Rlp is not valid.") + } + } + + // todo: should check length before encoding, cause it may just be sha3 of data + fn encoded(&self) -> Bytes { + match *self { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::Extension(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::Branch(ref nodes, ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(n) => { stream.append(&n); }, + None => { stream.append_empty_data(); }, + } + stream.out() + }, + Node::Empty => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } + + fn encoded_and_added(&self, diff: &mut Diff) -> Bytes { + let mut stream = RlpStream::new(); + match *self { + Node::Leaf(ref slice, ref value) => { + stream.append_list(2); + stream.append(&slice.encoded(true)); + stream.append(value); + }, + Node::Extension(ref slice, ref raw_rlp) => { + stream.append_list(2); + stream.append(&slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + }, + Node::Branch(ref nodes, ref value) => { + stream.append_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(n) => { stream.append(&n); }, + None => { stream.append_empty_data(); }, + } + }, + Node::Empty => { + stream.append_empty_data(); + } + } + let node = stream.out(); + match node.len() { + 0 ... 31 => node, + _ => { + let mut stream = RlpStream::new(); + diff.new_node(node, &mut stream); + stream.out() + } + } + } +} + +//enum ValidationResult<'a> { + //Valid, + //Invalid { node: Node<'a>, depth: usize } +//} + pub struct TrieDB { db: Box, root: H256, @@ -157,6 +202,11 @@ impl fmt::Debug for TrieDB { } } +enum MaybeChanged<'a> { + Same(Node<'a>), + Changed(Bytes), +} + impl TrieDB { pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r } @@ -201,9 +251,9 @@ impl TrieDB { fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { match node { - Node::Leaf(slice, value) => try!(writeln!(f, "-{:?}: {:?}.", slice, value.pretty())), + Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())), Node::Extension(ref slice, ref item) => { - try!(write!(f, "-{:?}- ", slice)); + try!(write!(f, "'{:?} ", slice)); try!(self.fmt_all(self.get_node(item), f, deepness)); }, Node::Branch(ref nodes, ref value) => { @@ -220,7 +270,7 @@ impl TrieDB { Node::Empty => {}, n => { try!(self.fmt_indent(f, deepness + 1)); - try!(write!(f, "{:x}: ", i)); + try!(write!(f, "'{:x} ", i)); try!(self.fmt_all(n, f, deepness + 1)); } } @@ -253,7 +303,7 @@ impl TrieDB { } } - fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { + fn get_raw_or_lookup<'a, 'b>(&'a self, node: &'b [u8]) -> &'b [u8] where 'a: 'b { // check if its sha3 + len let r = Rlp::new(node); match r.is_data() && r.size() == 32 { @@ -276,9 +326,15 @@ impl TrieDB { trace!("DELETE: {:?}", key); // determine what the new root is, insert new nodes and remove old as necessary. let mut todo: Diff = Diff::new(); - let root_rlp = self.cleared(self.db.lookup(&self.root).expect("Trie root not found!"), key, &mut todo); - self.apply(todo); - self.set_root_rlp(&root_rlp); + match self.cleared_from_slice(self.db.lookup(&self.root).expect("Trie root not found!"), key, &mut todo) { + Some(root_rlp) => { + self.apply(todo); + self.set_root_rlp(&root_rlp); + }, + None => { + trace!("no change needed"); + } + } trace!("/"); } @@ -492,24 +548,159 @@ impl TrieDB { _ => panic!("Invalid RLP for node: {:?}", old.pretty()), } } + fn encoded(n: MaybeChanged) -> Bytes { + match n { + MaybeChanged::Same(n) => n.encoded(), + MaybeChanged::Changed(b) => b, + } + } + fn ensure_is_changed(n: MaybeChanged) -> MaybeChanged { + match n { + MaybeChanged::Same(n) => MaybeChanged::Changed(n.encoded()), + f => f, + } + } + fn fixed_indirection<'a>(n: Node<'a>, diff: &mut Diff) -> MaybeChanged<'a> { + match n { + Node::Extension(partial, payload) if payload.len() >= 32 => { + // make indirect + MaybeChanged::Changed(Node::Extension(partial, &Node::decoded(payload).encoded_and_added(diff)).encoded()) + }, + Node::Branch(nodes, node_value) => { + // check each child isn't too big + // TODO OPTIMISE - should really check at the point of (re-)constructing the branch. + for i in 0..16 { + if nodes[i].len() >= 32 { + let n = Node::decoded(nodes[i]).encoded_and_added(diff); + let mut new_nodes = nodes; + new_nodes[i] = &n; + return MaybeChanged::Changed(Node::Branch(new_nodes, node_value).encoded()) + } + } + MaybeChanged::Same(n) + } + _ => MaybeChanged::Same(n), + } + } + + /// Given a node `n` which may be in an _invalid state_, fix it such that it is then in a valid + /// state. + /// + /// _invalid state_ means: + /// - Branch node where there is only a single entry; + /// - Extension node followed by anything other than a Branch node. + /// - Extension node with a child which has too many bytes to be inline. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn fixed<'a, 'b>(&'a self, n: Node<'b>, diff: &mut Diff) -> MaybeChanged<'b> where 'a: 'b { + match n { + Node::Branch(nodes, node_value) => { + // if only a single value, transmute to leaf/extension and feed through fixed. + let mut index: [u8; 1] = [16; 1]; + // 0-15 -> index of a non-null branch + // 16 -> no non-null branch + // 17 -> multiple non-null branches + for i in 0..16 { + match (nodes[i] == NULL_RLP, index[0]) { + (false, _) => {}, + (true, 16) => index[0] = i as u8, + (true, _) => index[0] = 17, + } + } + match (index[0], node_value) { + (16, None) => panic!("Branch with no subvalues. Something went wrong."), + (0 ... 15, None) => { // one onward node + // transmute to extension. + // TODO: OPTIMISE: - don't call fixed again but put the right node in straight away here. + // call fixed again since the transmute may cause invalidity. + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&index[..], 1), nodes[index[0] as usize]), diff))) + }, + (16, Some(value)) => { // one leaf value + // transmute to leaf. + // call fixed again since the transmute may cause invalidity. + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Leaf(NibbleSlice::new(&b""[..]), value), diff))) + } + _ => { // onwards node(s) and/or leaf + // no transmute needed, but should still fix the indirection. + Self::fixed_indirection(Node::Branch(nodes, node_value), diff) + }, + } + }, + Node::Extension(partial, payload) => { + match Node::decoded(self.get_raw_or_lookup(payload)) { + Node::Extension(sub_partial, sub_payload) => { + // combine with node below + diff.delete_node_from_slice(payload); + MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Extension(NibbleSlice::new_composed(&partial, &sub_partial), sub_payload), diff))) + }, + Node::Leaf(sub_partial, sub_value) => { + // combine with node below + diff.delete_node_from_slice(payload); + MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Leaf(NibbleSlice::new_composed(&partial, &sub_partial), sub_value), diff))) + }, + // no change, might still have an oversize node inline - fix indirection + _ => Self::fixed_indirection(n, diff), + } + }, + // leaf or empty. no change. + n => { MaybeChanged::Same(n) } + } + } /// Determine the RLP of the node, assuming we're removing `partial` from the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; /// it will just return the new RLP that represents the new node. + /// `None` may be returned should no change be needed. /// /// The database will be updated so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. /// /// **This operation will not insert the new node nor destroy the original.** - fn cleared(&self, old: &[u8], partial: &NibbleSlice, _diff: &mut Diff) -> Bytes { - trace!("cleared (old: {:?}, partial: {:?})", old.pretty(), partial); + fn cleared_from_slice(&self, old: &[u8], partial: &NibbleSlice, diff: &mut Diff) -> Option { + self.cleared(Node::decoded(old), partial, diff) + } - unimplemented!(); + fn cleared(&self, n: Node, partial: &NibbleSlice, diff: &mut Diff) -> Option { + trace!("cleared (old: {:?}, partial: {:?})", n, partial); -/* match (Node::decoded(old)) { - - }*/ + match (n, partial.is_empty()) { + (Node::Empty, _) => None, + (Node::Branch(nodes, None), true) => { None }, + (Node::Branch(nodes, _), true) => Some(Self::encoded(self.fixed(Node::Branch(nodes, None), diff))), // matched as leaf-branch - give back fixed branch with it. + (Node::Branch(nodes, value), false) => { + // Branch with partial left - route, clear, fix. + let i: usize = partial.at(0) as usize; + self.cleared(self.get_node(nodes[i]), &partial.mid(1), diff).map(|new_payload| { + // downsteam node needed to be changed. + diff.delete_node_from_slice(nodes[i]); + // return fixed up new node. + let mut new_nodes = nodes; + new_nodes[i] = &new_payload; + Self::encoded(self.fixed(Node::Branch(new_nodes, value), diff)) + }) + }, + (Node::Leaf(node_partial, node_value), _) => { + match node_partial.common_prefix(partial) { + cp if cp == partial.len() => Some(Node::Empty.encoded()), // leaf to be deleted - delete it :) + cp => None, // anything else and the key doesn't exit - no change. + } + }, + (Node::Extension(node_partial, node_payload), _) => { + match node_partial.common_prefix(partial) { + cp if cp < partial.len() => None, // key in the middle of an extension - doesn't exist. + cp => { + // key at end of extension - skip, clear, fix + self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), diff).map(|new_payload| { + // downsteam node needed to be changed. + diff.delete_node_from_slice(node_payload); + // return fixed up new node. + Self::encoded(self.fixed(Node::Extension(node_partial, &new_payload), diff)) + }) + }, + } + }, + } } } @@ -544,6 +735,136 @@ mod tests { use rand::random; use bytes::ToPretty; + #[test] + fn playpen() { + env_logger::init().ok(); + + let mut t1 = TrieDB::new_memory(); + t1.insert(&[0x01], &[0]); + t1.insert(&[0x01, 0x23], &[1]); + t1.insert(&[0x01, 0x34], &[2]); + t1.remove(&[0x01]); + let mut t2 = TrieDB::new_memory(); + t2.insert(&[0x01, 0x23], &[1]); + t2.insert(&[0x01, 0x34], &[2]); + /*if t1.root() != t2.root()*/ { + trace!("{:?}", t1); + trace!("{:?}", t2); + } + } + + #[test] + fn init() { + let t = TrieDB::new_memory(); + assert_eq!(*t.root(), SHA3_NULL_RLP); + assert!(t.is_empty()); + } + + #[test] + fn insert_on_empty() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); + } + + #[test] + fn remove_to_empty() { + } + + #[test] + fn insert_replace_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); + assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); + } + + #[test] + fn insert_make_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) + ])); + } + + #[test] + fn insert_into_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); + t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), + (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), + ])); + } + + #[test] + fn insert_value_into_branch_root() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[], &[0x0]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![], vec![0x0]), + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + ])); + } + + #[test] + fn insert_split_leaf() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), + (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), + ])); + } + + #[test] + fn insert_split_extenstion() { + let mut t = TrieDB::new_memory(); + t.insert(&[0x01, 0x23, 0x45], &[0x01]); + t.insert(&[0x01, 0xf3, 0x45], &[0x02]); + t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01, 0x23, 0x45], vec![0x01]), + (vec![0x01, 0xf3, 0x45], vec![0x02]), + (vec![0x01, 0xf3, 0xf5], vec![0x03]), + ])); + } + + #[test] + fn insert_big_value() { + let big_value0 = b"00000000000000000000000000000000"; + let big_value1 = b"11111111111111111111111111111111"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value0); + t.insert(&[0x11u8, 0x23], big_value1); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value0.to_vec()), + (vec![0x11u8, 0x23], big_value1.to_vec()) + ])); + } + + #[test] + fn insert_duplicate_value() { + let big_value = b"00000000000000000000000000000000"; + + let mut t = TrieDB::new_memory(); + t.insert(&[0x01u8, 0x23], big_value); + t.insert(&[0x11u8, 0x23], big_value); + assert_eq!(*t.root(), trie_root(vec![ + (vec![0x01u8, 0x23], big_value.to_vec()), + (vec![0x11u8, 0x23], big_value.to_vec()) + ])); + } + #[test] fn test_node_leaf() { let k = vec![0x20u8, 0x01, 0x23, 0x45]; @@ -743,128 +1064,4 @@ mod tests { test_all(v); } - - #[test] - fn playpen() { - env_logger::init().ok(); - - let big_value = b"00000000000000000000000000000000"; - - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], big_value); - t.insert(&[0x11u8, 0x23], big_value); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], big_value.to_vec()), - (vec![0x11u8, 0x23], big_value.to_vec()) - ])); - } - - #[test] - fn init() { - let t = TrieDB::new_memory(); - assert_eq!(*t.root(), SHA3_NULL_RLP); - assert!(t.is_empty()); - } - - #[test] - fn insert_on_empty() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); - } - - #[test] - fn insert_replace_root() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); - assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); - } - - #[test] - fn insert_make_branch_root() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x11u8, 0x23], vec![0x11u8, 0x23]) - ])); - } - - #[test] - fn insert_into_branch_root() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); - t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), - (vec![0xf1u8, 0x23], vec![0xf1u8, 0x23]), - ])); - } - - #[test] - fn insert_value_into_branch_root() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[], &[0x0]); - assert_eq!(*t.root(), trie_root(vec![ - (vec![], vec![0x0]), - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - ])); - } - - #[test] - fn insert_split_leaf() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), - (vec![0x01u8, 0x34], vec![0x01u8, 0x34]), - ])); - } - - #[test] - fn insert_split_extenstion() { - let mut t = TrieDB::new_memory(); - t.insert(&[0x01, 0x23, 0x45], &[0x01]); - t.insert(&[0x01, 0xf3, 0x45], &[0x02]); - t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01, 0x23, 0x45], vec![0x01]), - (vec![0x01, 0xf3, 0x45], vec![0x02]), - (vec![0x01, 0xf3, 0xf5], vec![0x03]), - ])); - } - - #[test] - fn insert_big_value() { - let big_value0 = b"00000000000000000000000000000000"; - let big_value1 = b"11111111111111111111111111111111"; - - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], big_value0); - t.insert(&[0x11u8, 0x23], big_value1); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], big_value0.to_vec()), - (vec![0x11u8, 0x23], big_value1.to_vec()) - ])); - } - - #[test] - fn insert_duplicate_value() { - let big_value = b"00000000000000000000000000000000"; - - let mut t = TrieDB::new_memory(); - t.insert(&[0x01u8, 0x23], big_value); - t.insert(&[0x11u8, 0x23], big_value); - assert_eq!(*t.root(), trie_root(vec![ - (vec![0x01u8, 0x23], big_value.to_vec()), - (vec![0x11u8, 0x23], big_value.to_vec()) - ])); - } } - diff --git a/src/triehash.rs b/src/triehash.rs index 778dc7f17..d31cec473 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -381,7 +381,7 @@ mod tests { #[test] fn test_triehash_json_trietest_json() { - let data = include_bytes!("../tests/TrieTests/trietest.json"); +/* let data = include_bytes!("../tests/TrieTests/trietest.json"); let s = String::from_bytes(data).unwrap(); let json = Json::from_str(&s).unwrap(); @@ -390,7 +390,7 @@ mod tests { for (key, value) in obj.iter() { println!("running test: {}", key); } - assert!(false); + assert!(false);*/ } } From 1b41b96dd109c072b75cc08132d358ad78d5387e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 23:00:36 +0100 Subject: [PATCH 200/379] Clear up warnings. --- src/trie.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index b205eb842..1dde1993c 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -548,6 +548,7 @@ impl TrieDB { _ => panic!("Invalid RLP for node: {:?}", old.pretty()), } } + fn encoded(n: MaybeChanged) -> Bytes { match n { MaybeChanged::Same(n) => n.encoded(), @@ -555,12 +556,6 @@ impl TrieDB { } } - fn ensure_is_changed(n: MaybeChanged) -> MaybeChanged { - match n { - MaybeChanged::Same(n) => MaybeChanged::Changed(n.encoded()), - f => f, - } - } fn fixed_indirection<'a>(n: Node<'a>, diff: &mut Diff) -> MaybeChanged<'a> { match n { Node::Extension(partial, payload) if payload.len() >= 32 => { @@ -666,7 +661,7 @@ impl TrieDB { match (n, partial.is_empty()) { (Node::Empty, _) => None, - (Node::Branch(nodes, None), true) => { None }, + (Node::Branch(_, None), true) => { None }, (Node::Branch(nodes, _), true) => Some(Self::encoded(self.fixed(Node::Branch(nodes, None), diff))), // matched as leaf-branch - give back fixed branch with it. (Node::Branch(nodes, value), false) => { // Branch with partial left - route, clear, fix. @@ -680,16 +675,16 @@ impl TrieDB { Self::encoded(self.fixed(Node::Branch(new_nodes, value), diff)) }) }, - (Node::Leaf(node_partial, node_value), _) => { + (Node::Leaf(node_partial, _), _) => { match node_partial.common_prefix(partial) { cp if cp == partial.len() => Some(Node::Empty.encoded()), // leaf to be deleted - delete it :) - cp => None, // anything else and the key doesn't exit - no change. + _ => None, // anything else and the key doesn't exit - no change. } }, (Node::Extension(node_partial, node_payload), _) => { match node_partial.common_prefix(partial) { cp if cp < partial.len() => None, // key in the middle of an extension - doesn't exist. - cp => { + _ => { // key at end of extension - skip, clear, fix self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), diff).map(|new_payload| { // downsteam node needed to be changed. From 8c9340d3cf41f67beca926533130800fcd20ba69 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 23:01:32 +0100 Subject: [PATCH 201/379] Remove unused uses. --- src/triehash.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/triehash.rs b/src/triehash.rs index d31cec473..c12971112 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -278,10 +278,7 @@ fn test_hex_prefix_encode() { #[cfg(test)] mod tests { use std::str::FromStr; - use std::collections::BTreeMap; use rustc_serialize::hex::FromHex; - use rustc_serialize::json::Json; - use bytes::*; use hash::*; use triehash::*; From 3f31d617505ffc75c66a46f9d2319a4e0ba70d86 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Wed, 2 Dec 2015 23:29:22 +0100 Subject: [PATCH 202/379] Update README.md --- json-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json-tests/README.md b/json-tests/README.md index 170cd588d..9e6915ca3 100644 --- a/json-tests/README.md +++ b/json-tests/README.md @@ -3,7 +3,7 @@ Cause it's very hard to write generic json test files, each subdirectory should follow its own convention. BUT all json files `within` same directory should be consistent. -### Test file should always contain a single file with input and output. +### Test files should always contain a single test with input and output. ```json { From 46d841f81f7eda103eb9fc913065b62720efb000 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 2 Dec 2015 23:35:34 +0100 Subject: [PATCH 203/379] removed redundant lifetime --- src/trie.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index 1dde1993c..3c8527424 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -303,7 +303,7 @@ impl TrieDB { } } - fn get_raw_or_lookup<'a, 'b>(&'a self, node: &'b [u8]) -> &'b [u8] where 'a: 'b { + fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { // check if its sha3 + len let r = Rlp::new(node); match r.is_data() && r.size() == 32 { From f20665c41a9868f5eb055c7fa5d4dfb19168c92e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 2 Dec 2015 23:50:14 +0100 Subject: [PATCH 204/379] Fix test. --- src/trie.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/trie.rs b/src/trie.rs index 1dde1993c..64af55a5e 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -887,9 +887,11 @@ mod tests { #[test] fn test_node_empty_branch() { - let branch = Node::Branch([&b""[..]; 16], None); + let null_rlp = NULL_RLP; + let branch = Node::Branch([&null_rlp; 16], None); let rlp = branch.encoded(); let branch2 = Node::decoded(&rlp); + println!("{:?}", rlp); assert_eq!(branch, branch2); } From 8b8a2e39e2bc66f0550d1b7c567ac6091774fe3e Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 00:29:31 +0100 Subject: [PATCH 205/379] trie test failing --- json-tests/src/trie.rs | 24 ++++++++++-------------- src/trie.rs | 21 +++++++++++++++++++++ src/triehash.rs | 4 ++-- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/json-tests/src/trie.rs b/json-tests/src/trie.rs index f6d4eaae1..827bf8c9a 100644 --- a/json-tests/src/trie.rs +++ b/json-tests/src/trie.rs @@ -4,7 +4,7 @@ use rustc_serialize::*; use rustc_serialize::hex::FromHex; use super::{JsonTest, JsonLoader}; -pub enum OperationType { +enum OperationType { Insert, Remove } @@ -26,10 +26,9 @@ struct RawOperation { value: Option } -pub struct Operation { - pub operation: OperationType, - pub key: Vec, - pub value: Option> +pub enum Operation { + Insert(Vec, Vec), + Remove(Vec) } fn hex_or_string(s: &str) -> Vec { @@ -41,12 +40,9 @@ fn hex_or_string(s: &str) -> Vec { impl Into for RawOperation { fn into(self) -> Operation { - Operation { - operation: self.operation, - key: hex_or_string(&self.key), - value: self.value.map(|v| { - hex_or_string(&v) - }) + match self.operation { + OperationType::Insert => Operation::Insert(hex_or_string(&self.key), hex_or_string(&self.value.unwrap())), + OperationType::Remove => Operation::Remove(hex_or_string(&self.key)) } } } @@ -96,9 +92,9 @@ impl JsonTest for TriehashTest { self.trietest.input() .into_iter() .fold(HashMap::new(), | mut map, o | { - match o.operation { - OperationType::Insert => map.insert(o.key, o.value.unwrap()), - OperationType::Remove => map.remove(&o.key) + match o { + Operation::Insert(k, v) => map.insert(k, v), + Operation::Remove(k) => map.remove(&k) }; map }) diff --git a/src/trie.rs b/src/trie.rs index 85312a58c..11e87c302 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -721,8 +721,11 @@ impl Trie for TrieDB { #[cfg(test)] mod tests { + extern crate json_tests; + use self::json_tests::*; use rustc_serialize::hex::FromHex; use triehash::*; + use hash::*; use super::*; use nibbleslice::*; use rlp; @@ -1061,4 +1064,22 @@ mod tests { test_all(v); } + + #[test] + fn test_trie_json() { + println!("Json trie test: "); + execute_tests_from_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { + println!("file: {}", file); + + let mut t = TrieDB::new_memory(); + for operation in input.into_iter() { + match operation { + trie::Operation::Insert(key, value) => t.insert(&key, &value), + trie::Operation::Remove(key) => t.remove(&key) + } + } + + assert_eq!(*t.root(), H256::from_slice(&output)); + }); + } } diff --git a/src/triehash.rs b/src/triehash.rs index 6cfa2103d..93d21aacc 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -283,7 +283,7 @@ mod tests { use triehash::*; #[test] - fn test_trie_out_of_order() { + fn test_triehash_out_of_order() { assert!(trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]), (vec![0x81u8, 0x23], vec![0x81u8, 0x23]), @@ -297,7 +297,7 @@ mod tests { } #[test] - fn test_trie_json() { + fn test_triehash_json() { execute_tests_from_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { println!("file: {}, output: {:?}", file, output); assert_eq!(trie_root(input), H256::from_slice(&output)); From b99cefb9d6a302382689dff29940a57b0ee48d64 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Dec 2015 00:32:58 +0100 Subject: [PATCH 206/379] Fix for node removal from trie. --- src/trie.rs | 67 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 85312a58c..550d7271e 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -589,29 +589,38 @@ impl TrieDB { /// /// **This operation will not insert the new node nor destroy the original.** fn fixed<'a, 'b>(&'a self, n: Node<'b>, diff: &mut Diff) -> MaybeChanged<'b> where 'a: 'b { + trace!("fixed node={:?}", n); match n { Node::Branch(nodes, node_value) => { // if only a single value, transmute to leaf/extension and feed through fixed. - let mut index: [u8; 1] = [16; 1]; + #[derive(Debug)] + enum UsedIndex { + None, + One(usize), + Many, + }; + let mut used_index = UsedIndex::None; // 0-15 -> index of a non-null branch // 16 -> no non-null branch // 17 -> multiple non-null branches for i in 0..16 { - match (nodes[i] == NULL_RLP, index[0]) { - (false, _) => {}, - (true, 16) => index[0] = i as u8, - (true, _) => index[0] = 17, + match (nodes[i] == NULL_RLP, &used_index) { + (false, &UsedIndex::None) => used_index = UsedIndex::One(i), + (false, &UsedIndex::One(_)) => used_index = UsedIndex::Many, + (_, _) => {}, } } - match (index[0], node_value) { - (16, None) => panic!("Branch with no subvalues. Something went wrong."), - (0 ... 15, None) => { // one onward node + trace!("branch: used_index={:?}, node_value={:?}", used_index, node_value); + match (used_index, node_value) { + (UsedIndex::None, None) => panic!("Branch with no subvalues. Something went wrong."), + (UsedIndex::One(a), None) => { // one onward node // transmute to extension. // TODO: OPTIMISE: - don't call fixed again but put the right node in straight away here. // call fixed again since the transmute may cause invalidity. - MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&index[..], 1), nodes[index[0] as usize]), diff))) + let new_partial: [u8; 1] = [a as u8; 1]; + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&new_partial[..], 1), nodes[a as usize]), diff))) }, - (16, Some(value)) => { // one leaf value + (UsedIndex::None, Some(value)) => { // one leaf value // transmute to leaf. // call fixed again since the transmute may cause invalidity. MaybeChanged::Changed(Self::encoded(self.fixed(Node::Leaf(NibbleSlice::new(&b""[..]), value), diff))) @@ -657,42 +666,52 @@ impl TrieDB { } fn cleared(&self, n: Node, partial: &NibbleSlice, diff: &mut Diff) -> Option { - trace!("cleared (old: {:?}, partial: {:?})", n, partial); + trace!("cleared old={:?}, partial={:?})", n, partial); match (n, partial.is_empty()) { (Node::Empty, _) => None, (Node::Branch(_, None), true) => { None }, - (Node::Branch(nodes, _), true) => Some(Self::encoded(self.fixed(Node::Branch(nodes, None), diff))), // matched as leaf-branch - give back fixed branch with it. - (Node::Branch(nodes, value), false) => { + (Node::Branch(payloads, _), true) => Some(Self::encoded(self.fixed(Node::Branch(payloads, None), diff))), // matched as leaf-branch - give back fixed branch with it. + (Node::Branch(payloads, value), false) => { // Branch with partial left - route, clear, fix. let i: usize = partial.at(0) as usize; - self.cleared(self.get_node(nodes[i]), &partial.mid(1), diff).map(|new_payload| { + trace!("branch-with-partial node[{:?}]={:?}", i, payloads[i].pretty()); + self.cleared(self.get_node(payloads[i]), &partial.mid(1), diff).map(|new_payload| { + trace!("branch-new-payload={:?}; delete-old={:?}", new_payload.pretty(), payloads[i].pretty()); + // downsteam node needed to be changed. - diff.delete_node_from_slice(nodes[i]); + diff.delete_node_from_slice(payloads[i]); // return fixed up new node. - let mut new_nodes = nodes; - new_nodes[i] = &new_payload; - Self::encoded(self.fixed(Node::Branch(new_nodes, value), diff)) + let mut new_payloads = payloads; + new_payloads[i] = &new_payload; + Self::encoded(self.fixed(Node::Branch(new_payloads, value), diff)) }) }, (Node::Leaf(node_partial, _), _) => { + trace!("leaf partial={:?}", node_partial); match node_partial.common_prefix(partial) { - cp if cp == partial.len() => Some(Node::Empty.encoded()), // leaf to be deleted - delete it :) + cp if cp == partial.len() => { // leaf to be deleted - delete it :) + trace!("matched-prefix (cp={:?}): REPLACE-EMPTY", cp); + Some(Node::Empty.encoded()) + }, _ => None, // anything else and the key doesn't exit - no change. } }, (Node::Extension(node_partial, node_payload), _) => { + trace!("extension partial={:?}, payload={:?}", node_partial, node_payload.pretty()); match node_partial.common_prefix(partial) { - cp if cp < partial.len() => None, // key in the middle of an extension - doesn't exist. - _ => { + cp if cp == node_partial.len() => { + trace!("matching-prefix (cp={:?}): SKIP,CLEAR,FIXUP", cp); // key at end of extension - skip, clear, fix self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), diff).map(|new_payload| { + trace!("extension-new-payload={:?}; delete-old={:?}", new_payload.pretty(), node_payload.pretty()); // downsteam node needed to be changed. diff.delete_node_from_slice(node_payload); // return fixed up new node. Self::encoded(self.fixed(Node::Extension(node_partial, &new_payload), diff)) }) }, + _ => None, // key in the middle of an extension - doesn't exist. } }, } @@ -735,13 +754,13 @@ mod tests { env_logger::init().ok(); let mut t1 = TrieDB::new_memory(); - t1.insert(&[0x01], &[0]); - t1.insert(&[0x01, 0x23], &[1]); t1.insert(&[0x01, 0x34], &[2]); - t1.remove(&[0x01]); let mut t2 = TrieDB::new_memory(); + t2.insert(&[0x01], &[0]); t2.insert(&[0x01, 0x23], &[1]); t2.insert(&[0x01, 0x34], &[2]); + t2.remove(&[0x01]); + t2.remove(&[0x01, 0x23]); /*if t1.root() != t2.root()*/ { trace!("{:?}", t1); trace!("{:?}", t2); From 84cc7715b41ee08974a3b2d26f6b196ebebad5e1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Dec 2015 00:50:14 +0100 Subject: [PATCH 207/379] Fix trie - tests pass. --- src/trie.rs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index a1ffcfe6c..081fcfcff 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -558,19 +558,19 @@ impl TrieDB { fn fixed_indirection<'a>(n: Node<'a>, diff: &mut Diff) -> MaybeChanged<'a> { match n { - Node::Extension(partial, payload) if payload.len() >= 32 => { + Node::Extension(partial, payload) if payload.len() >= 32 && Rlp::new(payload).is_list() => { // make indirect MaybeChanged::Changed(Node::Extension(partial, &Node::decoded(payload).encoded_and_added(diff)).encoded()) }, - Node::Branch(nodes, node_value) => { + Node::Branch(payloads, value) => { // check each child isn't too big // TODO OPTIMISE - should really check at the point of (re-)constructing the branch. for i in 0..16 { - if nodes[i].len() >= 32 { - let n = Node::decoded(nodes[i]).encoded_and_added(diff); - let mut new_nodes = nodes; + if payloads[i].len() >= 32 && Rlp::new(payloads[i]).is_list() { + let n = Node::decoded(payloads[i]).encoded_and_added(diff); + let mut new_nodes = payloads; new_nodes[i] = &n; - return MaybeChanged::Changed(Node::Branch(new_nodes, node_value).encoded()) + return MaybeChanged::Changed(Node::Branch(new_nodes, value).encoded()) } } MaybeChanged::Same(n) @@ -596,7 +596,7 @@ impl TrieDB { #[derive(Debug)] enum UsedIndex { None, - One(usize), + One(u8), Many, }; let mut used_index = UsedIndex::None; @@ -605,7 +605,7 @@ impl TrieDB { // 17 -> multiple non-null branches for i in 0..16 { match (nodes[i] == NULL_RLP, &used_index) { - (false, &UsedIndex::None) => used_index = UsedIndex::One(i), + (false, &UsedIndex::None) => used_index = UsedIndex::One(i as u8), (false, &UsedIndex::One(_)) => used_index = UsedIndex::Many, (_, _) => {}, } @@ -617,7 +617,7 @@ impl TrieDB { // transmute to extension. // TODO: OPTIMISE: - don't call fixed again but put the right node in straight away here. // call fixed again since the transmute may cause invalidity. - let new_partial: [u8; 1] = [a as u8; 1]; + let new_partial: [u8; 1] = [a; 1]; MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&new_partial[..], 1), nodes[a as usize]), diff))) }, (UsedIndex::None, Some(value)) => { // one leaf value @@ -627,6 +627,7 @@ impl TrieDB { } _ => { // onwards node(s) and/or leaf // no transmute needed, but should still fix the indirection. + trace!("no-transmute: FIXINDIRECTION"); Self::fixed_indirection(Node::Branch(nodes, node_value), diff) }, } @@ -756,14 +757,16 @@ mod tests { fn playpen() { env_logger::init().ok(); + let big_value = b"00000000000000000000000000000000"; + let mut t1 = TrieDB::new_memory(); - t1.insert(&[0x01, 0x34], &[2]); + t1.insert(&[0x01, 0x23], &big_value.to_vec()); + t1.insert(&[0x01, 0x34], &big_value.to_vec()); let mut t2 = TrieDB::new_memory(); - t2.insert(&[0x01], &[0]); - t2.insert(&[0x01, 0x23], &[1]); - t2.insert(&[0x01, 0x34], &[2]); + t2.insert(&[0x01], &big_value.to_vec()); + t2.insert(&[0x01, 0x23], &big_value.to_vec()); + t2.insert(&[0x01, 0x34], &big_value.to_vec()); t2.remove(&[0x01]); - t2.remove(&[0x01, 0x23]); /*if t1.root() != t2.root()*/ { trace!("{:?}", t1); trace!("{:?}", t2); From 05f7e85d301c8400df7635c0236ca845d0237670 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 05:44:35 +0100 Subject: [PATCH 208/379] rlp tests, the beginning --- json-tests/json/rlp/README.md | 39 +++++++++++++++++++++++++ json-tests/json/rlp/catdog.json | 18 ++++++++++++ json-tests/src/lib.rs | 2 ++ json-tests/src/rlp.rs | 52 +++++++++++++++++++++++++++++++++ json-tests/src/trie.rs | 33 ++++----------------- json-tests/src/util.rs | 8 +++++ src/trie.rs | 2 +- 7 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 json-tests/json/rlp/README.md create mode 100644 json-tests/json/rlp/catdog.json create mode 100644 json-tests/src/rlp.rs create mode 100644 json-tests/src/util.rs diff --git a/json-tests/json/rlp/README.md b/json-tests/json/rlp/README.md new file mode 100644 index 000000000..89cb072c7 --- /dev/null +++ b/json-tests/json/rlp/README.md @@ -0,0 +1,39 @@ +# Rlp tests guideline + +Rlp can be tested in various ways. It can encode/decode a value or an array of values. Let's start with encoding. + +Each operation must have field: + +- `operation` - `append`, `append_list`, `append_empty` or `append_raw` + +Additionally `append` and `append_raw` must additionally define a `value` field: + +- `value` - data + +Also `append_raw` and `append_list` requires `len` field + +- `len` - integer + +### Encoding Test Example + +```json +{ + "input": + [ + { + "operation": "append_list", + "len": 2 + }, + { + "operation": "append", + "value": "cat" + }, + { + "operation": "append", + "value": "dog" + } + ] + "output": "0xc88363617183646f67" +} +``` + diff --git a/json-tests/json/rlp/catdog.json b/json-tests/json/rlp/catdog.json new file mode 100644 index 000000000..352b24892 --- /dev/null +++ b/json-tests/json/rlp/catdog.json @@ -0,0 +1,18 @@ +{ + "input": + [ + { + "operation": "append_list", + "len": 2 + }, + { + "operation": "append", + "value": "cat" + }, + { + "operation": "append", + "value": "dog" + } + ] + "output": "0xc88363617183646f67" +} diff --git a/json-tests/src/lib.rs b/json-tests/src/lib.rs index 8a800e8f9..0fb1b091a 100644 --- a/json-tests/src/lib.rs +++ b/json-tests/src/lib.rs @@ -8,7 +8,9 @@ use std::fs::File; use glob::glob; use rustc_serialize::*; +mod util; pub mod trie; +pub mod rlp; pub trait JsonTest: Sized { type Input; diff --git a/json-tests/src/rlp.rs b/json-tests/src/rlp.rs new file mode 100644 index 000000000..bf6131b5b --- /dev/null +++ b/json-tests/src/rlp.rs @@ -0,0 +1,52 @@ +//! json rlp tests +use rustc_serialize::*; +use super::{JsonTest, JsonLoader}; +use util::*; + +pub enum Operation { + Append(Vec), + AppendList(usize), + AppendRaw(Vec, usize), + AppendEmpty +} + +impl Into for json::Json { + fn into(self) -> Operation { + let obj = self.as_object().unwrap(); + match obj["operation"].as_string().unwrap().as_ref() { + "append" => Operation::Append(hex_or_string(obj["value"].as_string().unwrap())), + "append_list" => Operation::AppendList(obj["len"].as_u64().unwrap() as usize), + "append_raw" => Operation::AppendRaw(hex_or_string(obj["value"].as_string().unwrap()), obj["len"].as_u64().unwrap() as usize), + "append_empty" => Operation::AppendEmpty, + other => { panic!("Unsupported opertation: {}", other); } + } + } +} + +pub struct RlpStreamTest { + loader: JsonLoader +} + +impl JsonTest for RlpStreamTest { + type Input = Vec; + type Output = Vec; + + fn new(data: &[u8]) -> Self { + RlpStreamTest { + loader: JsonLoader::new(data) + } + } + + fn input(&self) -> Self::Input { + self.loader.input().as_array().unwrap() + .iter() + .cloned() + .map(|i| i.into()) + .collect() + } + + fn output(&self) -> Self::Output { + hex_or_string(self.loader.output().as_string().unwrap()) + } +} + diff --git a/json-tests/src/trie.rs b/json-tests/src/trie.rs index 827bf8c9a..bc65e9db9 100644 --- a/json-tests/src/trie.rs +++ b/json-tests/src/trie.rs @@ -1,27 +1,12 @@ //! json trie tests use std::collections::HashMap; use rustc_serialize::*; -use rustc_serialize::hex::FromHex; use super::{JsonTest, JsonLoader}; - -enum OperationType { - Insert, - Remove -} - -impl Decodable for OperationType { - fn decode(d: &mut D) -> Result where D: Decoder { - match try!(String::decode(d)).as_ref() { - "insert" => Ok(OperationType::Insert), - "remove" => Ok(OperationType::Remove), - other => panic!("invalid operation type: {}", other) - } - } -} +use util::*; #[derive(RustcDecodable)] struct RawOperation { - operation: OperationType, + operation: String, key: String, value: Option } @@ -31,18 +16,12 @@ pub enum Operation { Remove(Vec) } -fn hex_or_string(s: &str) -> Vec { - match s.starts_with("0x") { - true => s[2..].from_hex().unwrap(), - false => From::from(s) - } -} - impl Into for RawOperation { fn into(self) -> Operation { - match self.operation { - OperationType::Insert => Operation::Insert(hex_or_string(&self.key), hex_or_string(&self.value.unwrap())), - OperationType::Remove => Operation::Remove(hex_or_string(&self.key)) + match self.operation.as_ref() { + "insert" => Operation::Insert(hex_or_string(&self.key), hex_or_string(&self.value.unwrap())), + "remove" => Operation::Remove(hex_or_string(&self.key)), + other => panic!("invalid operation type: {}", other) } } } diff --git a/json-tests/src/util.rs b/json-tests/src/util.rs new file mode 100644 index 000000000..f9d1e4eab --- /dev/null +++ b/json-tests/src/util.rs @@ -0,0 +1,8 @@ +use rustc_serialize::hex::FromHex; + +pub fn hex_or_string(s: &str) -> Vec { + match s.starts_with("0x") { + true => s[2..].from_hex().unwrap(), + false => From::from(s) + } +} diff --git a/src/trie.rs b/src/trie.rs index 081fcfcff..9dd12d8a5 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -742,7 +742,7 @@ impl Trie for TrieDB { #[cfg(test)] mod tests { extern crate json_tests; - use self::json_tests::*; + use self::json_tests::{trie, execute_tests_from_directory}; use rustc_serialize::hex::FromHex; use triehash::*; use hash::*; From 9f9c508ebd9306aeb5e892c52583a71cf2dfc792 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 05:47:07 +0100 Subject: [PATCH 209/379] fixed example .json files --- json-tests/json/rlp/README.md | 2 +- json-tests/json/rlp/catdog.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/json-tests/json/rlp/README.md b/json-tests/json/rlp/README.md index 89cb072c7..ab8b01020 100644 --- a/json-tests/json/rlp/README.md +++ b/json-tests/json/rlp/README.md @@ -32,7 +32,7 @@ Also `append_raw` and `append_list` requires `len` field "operation": "append", "value": "dog" } - ] + ], "output": "0xc88363617183646f67" } ``` diff --git a/json-tests/json/rlp/catdog.json b/json-tests/json/rlp/catdog.json index 352b24892..ed4c02602 100644 --- a/json-tests/json/rlp/catdog.json +++ b/json-tests/json/rlp/catdog.json @@ -13,6 +13,6 @@ "operation": "append", "value": "dog" } - ] + ], "output": "0xc88363617183646f67" } From 84eb30a133653a9519be13b34e2e4e89234d984b Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 06:06:42 +0100 Subject: [PATCH 210/379] empty lists tests --- json-tests/json/rlp/catdog.json | 2 +- json-tests/json/rlp/empty_lists.json | 38 ++++++++++++++++ src/rlp.rs | 24 ++++++++++ src/trie.rs | 65 ---------------------------- 4 files changed, 63 insertions(+), 66 deletions(-) create mode 100644 json-tests/json/rlp/empty_lists.json diff --git a/json-tests/json/rlp/catdog.json b/json-tests/json/rlp/catdog.json index ed4c02602..724329784 100644 --- a/json-tests/json/rlp/catdog.json +++ b/json-tests/json/rlp/catdog.json @@ -14,5 +14,5 @@ "value": "dog" } ], - "output": "0xc88363617183646f67" + "output": "0xc88363617483646f67" } diff --git a/json-tests/json/rlp/empty_lists.json b/json-tests/json/rlp/empty_lists.json new file mode 100644 index 000000000..5ac649c2c --- /dev/null +++ b/json-tests/json/rlp/empty_lists.json @@ -0,0 +1,38 @@ +{ + "input": + [ + { + "operation": "append_list", + "len": 3 + }, + { + "operation": "append_list", + "len": 0 + }, + { + "operation": "append_list", + "len": 1 + }, + { + "operation": "append_list", + "len": 0 + }, + { + "operation": "append_list", + "len": 2 + }, + { + "operation": "append_list", + "len": 0 + }, + { + "operation": "append_list", + "len": 1 + }, + { + "operation": "append_list", + "len": 0 + } + ], + "output": "0xc7c0c1c0c3c0c1c0" +} diff --git a/src/rlp.rs b/src/rlp.rs index b055632b1..8cb5f2b9b 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1098,6 +1098,9 @@ impl Encoder for BasicEncoder { #[cfg(test)] mod tests { + extern crate json_tests; + use self::json_tests::execute_tests_from_directory; + use self::json_tests::rlp as rlptest; use std::{fmt, cmp}; use std::str::FromStr; use rlp; @@ -1496,4 +1499,25 @@ mod tests { let view = View::new(&data); let _data_slice = view.offset(1).data(); } + + #[test] + fn test_rlp_json() { + println!("Json rlp test: "); + execute_tests_from_directory::("json-tests/json/rlp/*.json", &mut | file, input, output | { + println!("file: {}", file); + + let mut stream = RlpStream::new(); + for operation in input.into_iter() { + match operation { + rlptest::Operation::Append(ref v) => stream.append(v), + rlptest::Operation::AppendList(len) => stream.append_list(len), + rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len), + rlptest::Operation::AppendEmpty => stream.append_empty_data() + }; + } + + assert_eq!(stream.out(), output); + }); + } + } diff --git a/src/trie.rs b/src/trie.rs index 9dd12d8a5..674d22488 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -743,7 +743,6 @@ impl Trie for TrieDB { mod tests { extern crate json_tests; use self::json_tests::{trie, execute_tests_from_directory}; - use rustc_serialize::hex::FromHex; use triehash::*; use hash::*; use super::*; @@ -969,24 +968,6 @@ mod tests { //assert!(false); } - fn test_all(v: Vec<(Vec, Vec)>) { - let mut t = TrieDB::new_memory(); - - for i in 0..v.len() { - let key: &[u8]= &v[i].0; - let val: &[u8] = &v[i].1; - t.insert(&key, &val); - } - -// trace!("{:?}", t); -// println!("{:?}", t); - - // check lifetime -// let _q = t.at(&[b'd', b'o']).unwrap(); - - assert_eq!(*t.root(), trie_root(v)); - } - fn random_key() -> Vec { let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; let mut ret: Vec = Vec::new(); @@ -1041,52 +1022,6 @@ mod tests { t } - #[test] - fn test_at_dog() { - env_logger::init().ok(); - let v = vec![ - (From::from("do"), From::from("verb")), - (From::from("dog"), From::from("puppy")), - (From::from("doge"), From::from("coin")), - (From::from("horse"), From::from("stallion")), - ]; - - test_all(v); - } - - #[test] - fn test_more_data() { - let v = vec![ - - ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), - "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), - - ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), - "67706c2076330000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - - ("000000000000000000000000697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap(), - "6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap()), - - ("0000000000000000000000007ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap(), - "4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap()), - - ("000000000000000000000000ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap(), - "4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap()), - - ("4655474156000000000000000000000000000000000000000000000000000000".from_hex().unwrap(), - "7ef9e639e2733cb34e4dfc576d4b23f72db776b2".from_hex().unwrap()), - - ("4e616d6552656700000000000000000000000000000000000000000000000000".from_hex().unwrap(), - "ec4f34c97e43fbb2816cfd95e388353c7181dab1".from_hex().unwrap()), - - ("6f6f6f6820736f2067726561742c207265616c6c6c793f000000000000000000".from_hex().unwrap(), - "697c7b8c961b56f675d570498424ac8de1a918f6".from_hex().unwrap()) - - ]; - - test_all(v); - } - #[test] fn test_trie_json() { println!("Json trie test: "); From dc316dcfeb04619d659ac8768b62eaf595f1156f Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 06:13:27 +0100 Subject: [PATCH 211/379] additional simple json tests --- json-tests/json/rlp/empty.json | 9 +++++++++ json-tests/json/rlp/integer.json | 10 ++++++++++ 2 files changed, 19 insertions(+) create mode 100644 json-tests/json/rlp/empty.json create mode 100644 json-tests/json/rlp/integer.json diff --git a/json-tests/json/rlp/empty.json b/json-tests/json/rlp/empty.json new file mode 100644 index 000000000..19cbf4185 --- /dev/null +++ b/json-tests/json/rlp/empty.json @@ -0,0 +1,9 @@ +{ + "input": + [ + { + "operation": "append_empty" + } + ], + "output": "0x80" +} diff --git a/json-tests/json/rlp/integer.json b/json-tests/json/rlp/integer.json new file mode 100644 index 000000000..1effa4a1e --- /dev/null +++ b/json-tests/json/rlp/integer.json @@ -0,0 +1,10 @@ +{ + "input": + [ + { + "operation": "append", + "value": "0x0400" + } + ], + "output": "0x820400" +} From 611226c117ca773b9a3ef43cc4aa390e82e22cb9 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 11:36:20 +0100 Subject: [PATCH 212/379] small changes in tests --- json-tests/json/rlp/{ => stream}/catdog.json | 0 json-tests/json/rlp/{ => stream}/empty.json | 0 .../json/rlp/{ => stream}/empty_lists.json | 0 json-tests/json/rlp/{ => stream}/integer.json | 0 .../json/rlp/stream/list_of_empty_data.json | 22 +++++++++++++++++++ .../json/rlp/stream/list_of_empty_data2.json | 19 ++++++++++++++++ src/rlp.rs | 16 +++++++++++++- 7 files changed, 56 insertions(+), 1 deletion(-) rename json-tests/json/rlp/{ => stream}/catdog.json (100%) rename json-tests/json/rlp/{ => stream}/empty.json (100%) rename json-tests/json/rlp/{ => stream}/empty_lists.json (100%) rename json-tests/json/rlp/{ => stream}/integer.json (100%) create mode 100644 json-tests/json/rlp/stream/list_of_empty_data.json create mode 100644 json-tests/json/rlp/stream/list_of_empty_data2.json diff --git a/json-tests/json/rlp/catdog.json b/json-tests/json/rlp/stream/catdog.json similarity index 100% rename from json-tests/json/rlp/catdog.json rename to json-tests/json/rlp/stream/catdog.json diff --git a/json-tests/json/rlp/empty.json b/json-tests/json/rlp/stream/empty.json similarity index 100% rename from json-tests/json/rlp/empty.json rename to json-tests/json/rlp/stream/empty.json diff --git a/json-tests/json/rlp/empty_lists.json b/json-tests/json/rlp/stream/empty_lists.json similarity index 100% rename from json-tests/json/rlp/empty_lists.json rename to json-tests/json/rlp/stream/empty_lists.json diff --git a/json-tests/json/rlp/integer.json b/json-tests/json/rlp/stream/integer.json similarity index 100% rename from json-tests/json/rlp/integer.json rename to json-tests/json/rlp/stream/integer.json diff --git a/json-tests/json/rlp/stream/list_of_empty_data.json b/json-tests/json/rlp/stream/list_of_empty_data.json new file mode 100644 index 000000000..c5b898f25 --- /dev/null +++ b/json-tests/json/rlp/stream/list_of_empty_data.json @@ -0,0 +1,22 @@ +{ + "input": + [ + { + "operation": "append_list", + "len": 3 + }, + { + "operation": "append", + "value": "" + }, + { + "operation": "append", + "value": "" + }, + { + "operation": "append", + "value": "" + } + ], + "output": "0xc3808080" +} diff --git a/json-tests/json/rlp/stream/list_of_empty_data2.json b/json-tests/json/rlp/stream/list_of_empty_data2.json new file mode 100644 index 000000000..76043af91 --- /dev/null +++ b/json-tests/json/rlp/stream/list_of_empty_data2.json @@ -0,0 +1,19 @@ +{ + "input": + [ + { + "operation": "append_list", + "len": 3 + }, + { + "operation": "append_empty" + }, + { + "operation": "append_empty" + }, + { + "operation": "append_empty" + } + ], + "output": "0xc3808080" +} diff --git a/src/rlp.rs b/src/rlp.rs index 8cb5f2b9b..5f5c90590 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1335,6 +1335,20 @@ mod tests { 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); } + #[test] + fn rlp_stream_list4() { + let mut stream = RlpStream::new(); + stream.append_list(17); + let v: Vec = vec![]; + for _ in 0..17 { + stream.append(&v); + } + let out = stream.out(); + assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); + } + #[test] fn rlp_stream_list3() { let mut stream = RlpStream::new(); @@ -1503,7 +1517,7 @@ mod tests { #[test] fn test_rlp_json() { println!("Json rlp test: "); - execute_tests_from_directory::("json-tests/json/rlp/*.json", &mut | file, input, output | { + execute_tests_from_directory::("json-tests/json/rlp/stream/*.json", &mut | file, input, output | { println!("file: {}", file); let mut stream = RlpStream::new(); From 38a3650ba95741e9e60dcb442a53a1010b272aa9 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 3 Dec 2015 12:00:57 +0100 Subject: [PATCH 213/379] few additional json tests --- json-tests/json/.DS_Store | Bin 0 -> 6148 bytes json-tests/json/rlp/stream/bytestring0.json | 10 + json-tests/json/rlp/stream/bytestring1.json | 10 + json-tests/json/rlp/stream/bytestring7.json | 10 + json-tests/json/rlp/stream/catdog.json | 12 +- json-tests/json/rlp/stream/longlist.json | 521 ++++++++++++++++++++ json-tests/json/rlp/stream/longstring.json | 10 + 7 files changed, 563 insertions(+), 10 deletions(-) create mode 100644 json-tests/json/.DS_Store create mode 100644 json-tests/json/rlp/stream/bytestring0.json create mode 100644 json-tests/json/rlp/stream/bytestring1.json create mode 100644 json-tests/json/rlp/stream/bytestring7.json create mode 100644 json-tests/json/rlp/stream/longlist.json create mode 100644 json-tests/json/rlp/stream/longstring.json diff --git a/json-tests/json/.DS_Store b/json-tests/json/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5ddc63de7e78f0f09186cf46a2ff1cf2b9cae2aa GIT binary patch literal 6148 zcmeH~Jr2S!425mVfW*>~F$)La1`&c2Z~+8}2?+#Z&(V4QSt!h?LeG-@#ZIlZZ)j=} z(e?AR66r-`1~vIYIqf#4$mI7QhFYo8^+Vg;}!TM!kPMk^St>SKu2 zy&WuhT}`%Nw2S8Op?PPuDF&v|E?SVlv^p5502LT0&_&+c`M-sKoBszbOsN1B_%j7` zy4h{kc&R*FKVHx3`>fi!!9l+q;q4~?i5 Date: Thu, 3 Dec 2015 13:11:02 +0100 Subject: [PATCH 214/379] removed duplicate tests --- src/rlp.rs | 99 ++---------------------------------------------------- 1 file changed, 2 insertions(+), 97 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index 5f5c90590..076111b23 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -1259,7 +1259,7 @@ mod tests { run_encode_tests(tests); } - /// Vec is treated as a single value + /// Vec (Bytes) is treated as a single value #[test] fn encode_vector_u8() { let tests = vec![ @@ -1295,74 +1295,6 @@ mod tests { run_encode_tests(tests); } - #[test] - fn encode_bytes() { - let vec = vec![0u8]; - let slice: &[u8] = &vec; - let res = rlp::encode(&slice); - assert_eq!(res, vec![0u8]); - } - - #[test] - fn rlp_stream() { - let mut stream = RlpStream::new_list(2); - stream.append(&"cat").append(&"dog"); - let out = stream.out(); - assert_eq!(out, - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - } - - #[test] - fn rlp_stream_list() { - let mut stream = RlpStream::new_list(3); - stream.append_list(0); - stream.append_list(1).append_list(0); - stream.append_list(2).append_list(0).append_list(1).append_list(0); - let out = stream.out(); - assert_eq!(out, vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]); - } - - #[test] - fn rlp_stream_list2() { - let mut stream = RlpStream::new(); - stream.append_list(17); - for _ in 0..17 { - stream.append(&""); - } - let out = stream.out(); - assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); - } - - #[test] - fn rlp_stream_list4() { - let mut stream = RlpStream::new(); - stream.append_list(17); - let v: Vec = vec![]; - for _ in 0..17 { - stream.append(&v); - } - let out = stream.out(); - assert_eq!(out, vec![0xd1, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80]); - } - - #[test] - fn rlp_stream_list3() { - let mut stream = RlpStream::new(); - stream.append_list(17); - - let mut res = vec![0xf8, 0x44]; - for _ in 0..17 { - stream.append(&"aaa"); - res.extend(vec![0x83, b'a', b'a', b'a']); - } - let out = stream.out(); - assert_eq!(out, res); - } - struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { @@ -1372,7 +1304,7 @@ mod tests { } } - /// Vec is treated as a single value + /// Vec (Bytes) is treated as a single value #[test] fn decode_vector_u8() { let tests = vec![ @@ -1487,33 +1419,6 @@ mod tests { run_decode_tests(tests); } - #[test] - fn test_view() { - struct View<'a> { - bytes: &'a [u8] - } - - impl <'a, 'view> View<'a> where 'a: 'view { - fn new(bytes: &'a [u8]) -> View<'a> { - View { - bytes: bytes - } - } - - fn offset(&'view self, len: usize) -> View<'a> { - View::new(&self.bytes[len..]) - } - - fn data(&'view self) -> &'a [u8] { - self.bytes - } - } - - let data = vec![0, 1, 2, 3]; - let view = View::new(&data); - let _data_slice = view.offset(1).data(); - } - #[test] fn test_rlp_json() { println!("Json rlp test: "); From cbbe5ee0fe8562bcdeb6df9a9540fe07a89ea5e5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Dec 2015 14:56:39 +0100 Subject: [PATCH 215/379] trie node ref counter. good for testing. --- src/hashdb.rs | 4 ++++ src/memorydb.rs | 8 +++++++ src/overlaydb.rs | 15 ++++++++++++++ src/rlp.rs | 1 + src/trie.rs | 54 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+) diff --git a/src/hashdb.rs b/src/hashdb.rs index dd0973bb3..f893c8df8 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -1,7 +1,11 @@ use hash::*; use bytes::*; +use std::collections::HashMap; pub trait HashDB { + /// Get the keys in the database together with number of underlying references. + fn keys(&self) -> HashMap; + /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. /// diff --git a/src/memorydb.rs b/src/memorydb.rs index 85fdc7c19..8c7eaff2c 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -116,6 +116,10 @@ impl MemoryDB { } self.data.get(key).unwrap() } + + pub fn raw_keys(&self) -> HashMap { + self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect::>() + } } impl HashDB for MemoryDB { @@ -126,6 +130,10 @@ impl HashDB for MemoryDB { } } + fn keys(&self) -> HashMap { + self.data.iter().filter_map(|(k, v)| if v.1 > 0 {Some((k.clone(), v.1 as u32))} else {None} ).collect::>() + } + fn exists(&self, key: &H256) -> bool { match self.data.get(key) { Some(&(_, x)) if x > 0 => true, diff --git a/src/overlaydb.rs b/src/overlaydb.rs index d69afa0f1..78ca67d01 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -9,6 +9,7 @@ use memorydb::*; use std::ops::*; use std::sync::*; use std::env; +use std::collections::HashMap; use rocksdb::{DB, Writable}; #[derive(Clone)] @@ -135,6 +136,20 @@ impl OverlayDB { } impl HashDB for OverlayDB { + fn keys(&self) -> HashMap { + let mut ret: HashMap = HashMap::new(); + for (key, _) in self.backing.iterator().from_start() { + let h = H256::from_slice(key.deref()); + let r = self.payload(&h).unwrap().1; + ret.insert(h, r); + } + + for (key, refs) in self.overlay.raw_keys().into_iter() { + let refs = *ret.get(&key).unwrap_or(&0u32) as i32 + refs as i32; + ret.insert(key, refs as u32); + } + ret + } fn lookup(&self, key: &H256) -> Option<&[u8]> { // return ok if positive; if negative, check backing - might be enough references there to make // it positive again. diff --git a/src/rlp.rs b/src/rlp.rs index b055632b1..b875e61c5 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -125,6 +125,7 @@ impl<'a> From> for UntrustedRlp<'a> { } } +#[derive(Debug)] pub enum Prototype { Null, Data(usize), diff --git a/src/trie.rs b/src/trie.rs index 081fcfcff..3d825c707 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -8,6 +8,7 @@ use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; +use std::collections::HashMap; //use log::*; @@ -238,6 +239,52 @@ impl TrieDB { } } + pub fn keys(&self) -> Vec { + let mut ret: Vec = Vec::new(); + ret.push(self.root.clone()); + self.accumulate_keys(self.root_node(), &mut ret); + ret + } + + fn accumulate_keys(&self, node: Node, acc: &mut Vec) { + let mut handle_payload = |payload| { + let p = Rlp::new(payload); + if p.is_data() && p.size() == 32 { + acc.push(H256::decode(&p)); + } + + self.accumulate_keys(self.get_node(payload), acc); + }; + + match node { + Node::Extension(_, payload) => handle_payload(payload), + Node::Branch(payloads, _) => for payload in payloads.iter() { handle_payload(payload) }, + _ => {}, + } + } + + fn to_map(hashes: Vec) -> HashMap { + let mut r: HashMap = HashMap::new(); + for h in hashes.into_iter() { + let c = *r.get(&h).unwrap_or(&0); + r.insert(h, c + 1); + } + r + } + + pub fn db_items_remaining(&self) -> HashMap { + let mut ret = self.db().keys(); + for (k, v) in Self::to_map(self.keys()).into_iter() { + let old = *ret.get(&k).expect("Node in trie is not in database!"); + assert!(old >= v); + match old > v { + true => ret.insert(k, old - v), + _ => ret.remove(&k), + }; + } + ret + } + fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { for _ in 0..size { try!(write!(f, " ")); @@ -245,6 +292,10 @@ impl TrieDB { Ok(()) } + fn root_node(&self) -> Node { + Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) + } + fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { Node::decoded(self.get_raw_or_lookup(node)) } @@ -762,11 +813,14 @@ mod tests { let mut t1 = TrieDB::new_memory(); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); + trace!("keys remaining {:?}", t1.db_items_remaining()); + assert!(t1.db_items_remaining().is_empty()); let mut t2 = TrieDB::new_memory(); t2.insert(&[0x01], &big_value.to_vec()); t2.insert(&[0x01, 0x23], &big_value.to_vec()); t2.insert(&[0x01, 0x34], &big_value.to_vec()); t2.remove(&[0x01]); + assert!(t2.db_items_remaining().is_empty()); /*if t1.root() != t2.root()*/ { trace!("{:?}", t1); trace!("{:?}", t2); From 2e3750323a3f2317f7abcf0ad3ec6e6797daced5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Dec 2015 15:11:40 +0100 Subject: [PATCH 216/379] Bugfixes; Move discovery draft to a module --- .gitignore | 3 + src/crypto.rs | 27 +++-- src/network/connection.rs | 160 +++++++++++++++++++---------- src/network/discovery.rs | 204 ++++++++++++++++++++++++++++++++++++ src/network/handshake.rs | 33 +++--- src/network/host.rs | 210 +++----------------------------------- src/network/mod.rs | 1 + src/network/session.rs | 16 +-- src/triehash.rs | 52 +++++----- 9 files changed, 396 insertions(+), 310 deletions(-) create mode 100644 src/network/discovery.rs diff --git a/.gitignore b/.gitignore index eabd0a44e..e651c6c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ Cargo.lock # Vim *.swp + +# GDB +*.gdb_history diff --git a/src/crypto.rs b/src/crypto.rs index 0ffd0f876..b7422b9b9 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -38,8 +38,8 @@ impl From<::std::io::Error> for CryptoError { #[derive(Debug, PartialEq, Eq)] /// secp256k1 Key pair /// -/// Use `create()` to create a new random key pair. -/// +/// Use `create()` to create a new random key pair. +/// /// # Example /// ```rust /// extern crate ethcore_util; @@ -206,11 +206,11 @@ pub mod ecies { use ::rcrypto::hmac::Hmac; use ::rcrypto::mac::Mac; - let meta_len = encrypted.len() - (1 + 64 + 16 + 32); + let meta_len = 1 + 64 + 16 + 32; if encrypted.len() < meta_len || encrypted[0] < 2 || encrypted[0] > 4 { return Err(CryptoError::InvalidMessage); //invalid message: publickey } - + let e = &encrypted[1..]; let p = Public::from_slice(&e[0..64]); let z = try!(ecdh::agree(secret, &p)); @@ -224,14 +224,14 @@ pub mod ecies { hasher.result(&mut mkey); let clen = encrypted.len() - meta_len; - let cypher_with_iv = &e[64..(64+16+clen)]; - let cypher_iv = &cypher_with_iv[0..16]; - let cypher_no_iv = &cypher_with_iv[16..]; + let cipher_with_iv = &e[64..(64+16+clen)]; + let cipher_iv = &cipher_with_iv[0..16]; + let cipher_no_iv = &cipher_with_iv[16..]; let msg_mac = &e[(64+16+clen)..]; // Verify tag let mut hmac = Hmac::new(Sha256::new(), &mkey); - hmac.input(cypher_iv); + hmac.input(cipher_with_iv); let mut mac = H256::new(); hmac.raw_result(&mut mac); if &mac[..] != msg_mac { @@ -239,7 +239,7 @@ pub mod ecies { } let mut msg = vec![0u8; clen]; - aes::decrypt(ekey, &H128::new(), cypher_no_iv, &mut msg[..]); + aes::decrypt(ekey, cipher_iv, cipher_no_iv, &mut msg[..]); Ok(msg) } @@ -266,19 +266,18 @@ pub mod ecies { } pub mod aes { - use hash::*; use ::rcrypto::blockmodes::*; use ::rcrypto::aessafe::*; use ::rcrypto::symmetriccipher::*; use ::rcrypto::buffer::*; - pub fn encrypt(k: &[u8], iv: &H128, plain: &[u8], dest: &mut [u8]) { - let mut encryptor = CtrMode::new(AesSafe128Encryptor::new(k), iv[..].to_vec()); + pub fn encrypt(k: &[u8], iv: &[u8], plain: &[u8], dest: &mut [u8]) { + let mut encryptor = CtrMode::new(AesSafe128Encryptor::new(k), iv.to_vec()); encryptor.encrypt(&mut RefReadBuffer::new(plain), &mut RefWriteBuffer::new(dest), true).expect("Invalid length or padding"); } - pub fn decrypt(k: &[u8], iv: &H128, encrypted: &[u8], dest: &mut [u8]) { - let mut encryptor = CtrMode::new(AesSafe128Encryptor::new(k), iv[..].to_vec()); + pub fn decrypt(k: &[u8], iv: &[u8], encrypted: &[u8], dest: &mut [u8]) { + let mut encryptor = CtrMode::new(AesSafe128Encryptor::new(k), iv.to_vec()); encryptor.decrypt(&mut RefReadBuffer::new(encrypted), &mut RefWriteBuffer::new(dest), true).expect("Invalid length or padding"); } } diff --git a/src/network/connection.rs b/src/network/connection.rs index e3a935dde..93a92daa7 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -21,7 +21,7 @@ const ENCRYPTED_HEADER_LEN: usize = 32; pub struct Connection { pub token: Token, - pub socket: TcpStream, + pub socket: TcpStream, rec_buf: Bytes, rec_size: usize, send_queue: VecDeque>, @@ -61,7 +61,7 @@ impl Connection { } let max = self.rec_size - self.rec_buf.len(); // resolve "multiple applicable items in scope [E0034]" error - let sock_ref = ::by_ref(&mut self.socket); + let sock_ref = ::by_ref(&mut self.socket); match sock_ref.take(max as u64).try_read_buf(&mut self.rec_buf) { Ok(Some(_)) if self.rec_buf.len() == self.rec_size => { self.rec_size = 0; @@ -71,14 +71,14 @@ impl Connection { Err(e) => Err(e), } } - + pub fn send(&mut self, data: Bytes) { //TODO: take ownership version if data.len() != 0 { self.send_queue.push_back(Cursor::new(data)); } - if !self.interest.is_writable() { - self.interest.insert(EventSet::writable()); - } + if !self.interest.is_writable() { + self.interest.insert(EventSet::writable()); + } } pub fn writable(&mut self) -> io::Result { @@ -98,36 +98,41 @@ impl Connection { Ok(WriteStatus::Ongoing) }, Ok(_) if (buf.position() as usize) == send_size => { - self.interest.remove(EventSet::writable()); Ok(WriteStatus::Complete) }, Ok(_) => { panic!("Wrote past buffer");}, Err(e) => Err(e) } - }.and_then(|r| if r == WriteStatus::Complete { + }.and_then(|r| { + if r == WriteStatus::Complete { self.send_queue.pop_front(); - Ok(r) + }; + if self.send_queue.is_empty() { + self.interest.remove(EventSet::writable()); } - else { Ok(r) } - ) + else { + self.interest.insert(EventSet::writable()); + } + Ok(r) + }) } - pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection register; token={:?}", self.token); - self.interest.insert(EventSet::readable()); - event_loop.register_opt(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { - error!("Failed to reregister {:?}, {:?}", self.token, e); - Err(e) - }) - } + pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + trace!(target: "net", "connection register; token={:?}", self.token); + self.interest.insert(EventSet::readable()); + event_loop.register_opt(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + error!("Failed to reregister {:?}, {:?}", self.token, e); + Err(e) + }) + } - pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { - trace!(target: "net", "connection reregister; token={:?}", self.token); - event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { - error!("Failed to reregister {:?}, {:?}", self.token, e); - Err(e) - }) - } + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + trace!(target: "net", "connection reregister; token={:?}", self.token); + event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + error!("Failed to reregister {:?}, {:?}", self.token, e); + Err(e) + }) + } } pub struct Packet { @@ -142,9 +147,9 @@ enum EncryptedConnectionState { pub struct EncryptedConnection { connection: Connection, - encoder: CtrMode, - decoder: CtrMode, - mac_encoder: EcbEncryptor>, + encoder: CtrMode, + decoder: CtrMode, + mac_encoder: EcbEncryptor>, egress_mac: Keccak, ingress_mac: Keccak, read_state: EncryptedConnectionState, @@ -169,22 +174,24 @@ impl EncryptedConnection { shared.copy_to(&mut key_material[0..32]); nonce_material.sha3_into(&mut key_material[32..64]); key_material.sha3().copy_to(&mut key_material[32..64]); + key_material.sha3().copy_to(&mut key_material[32..64]); let iv = vec![0u8; 16]; - let encoder = CtrMode::new(AesSafe128Encryptor::new(&key_material[32..64]), iv); + let encoder = CtrMode::new(AesSafe256Encryptor::new(&key_material[32..64]), iv); let iv = vec![0u8; 16]; - let decoder = CtrMode::new(AesSafe128Encryptor::new(&key_material[32..64]), iv); + let decoder = CtrMode::new(AesSafe256Encryptor::new(&key_material[32..64]), iv); key_material.sha3().copy_to(&mut key_material[32..64]); - let mac_encoder = EcbEncryptor::new(AesSafe128Encryptor::new(&key_material[32..64]), NoPadding); - + let mac_encoder = EcbEncryptor::new(AesSafe256Encryptor::new(&key_material[32..64]), NoPadding); + println!("SESSION key: {}", H256::from_slice(&key_material[32..64]).hex()); + let mut egress_mac = Keccak::new_keccak256(); let mut mac_material = &H256::from_slice(&key_material[32..64]) ^ &handshake.remote_nonce; egress_mac.update(&mac_material); egress_mac.update(if handshake.originated { &handshake.auth_cipher } else { &handshake.ack_cipher }); - + let mut ingress_mac = Keccak::new_keccak256(); - mac_material = &(&mac_material ^ &handshake.remote_nonce) ^ &handshake.nonce; + mac_material = &H256::from_slice(&key_material[32..64]) ^ &handshake.nonce; ingress_mac.update(&mac_material); ingress_mac.update(if handshake.originated { &handshake.ack_cipher } else { &handshake.auth_cipher }); @@ -203,6 +210,7 @@ impl EncryptedConnection { } pub fn send_packet(&mut self, payload: &[u8]) -> Result<(), Error> { + println!("HEADER"); let mut header = RlpStream::new(); let len = payload.len() as usize; header.append_raw(&[(len >> 16) as u8, (len >> 8) as u8, len as u8], 1); @@ -214,14 +222,15 @@ impl EncryptedConnection { let mut packet = vec![0u8; (32 + payload.len() + padding + 16)]; self.encoder.encrypt(&mut RefReadBuffer::new(&header), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding"); - self.egress_mac.update(&packet[0..16]); + EncryptedConnection::update_mac(&mut self.egress_mac, &mut self.mac_encoder, &packet[0..16]); self.egress_mac.clone().finalize(&mut packet[16..32]); - self.encoder.encrypt(&mut RefReadBuffer::new(&payload), &mut RefWriteBuffer::new(&mut packet[32..(32 + len)]), padding == 0).expect("Invalid length or padding"); + self.encoder.encrypt(&mut RefReadBuffer::new(&payload), &mut RefWriteBuffer::new(&mut packet[32..(32 + len)]), padding == 0).expect("Invalid length or padding"); if padding != 0 { let pad = [08; 16]; - self.encoder.encrypt(&mut RefReadBuffer::new(&pad[0..padding]), &mut RefWriteBuffer::new(&mut packet[(32 + len)..(32 + len + padding)]), true).expect("Invalid length or padding"); + self.encoder.encrypt(&mut RefReadBuffer::new(&pad[0..padding]), &mut RefWriteBuffer::new(&mut packet[(32 + len)..(32 + len + padding)]), true).expect("Invalid length or padding"); } self.egress_mac.update(&packet[32..(32 + len + padding)]); + EncryptedConnection::update_mac(&mut self.egress_mac, &mut self.mac_encoder, &[0u8; 0]); self.egress_mac.clone().finalize(&mut packet[(32 + len + padding)..]); self.connection.send(packet); Ok(()) @@ -231,19 +240,19 @@ impl EncryptedConnection { if header.len() != ENCRYPTED_HEADER_LEN { return Err(Error::Auth); } - self.ingress_mac.update(header); + EncryptedConnection::update_mac(&mut self.ingress_mac, &mut self.mac_encoder, &header[0..16]); let mac = &header[16..]; - let mut expected = H128::new(); + let mut expected = H256::new(); self.ingress_mac.clone().finalize(&mut expected); - if mac != &expected[..] { + if mac != &expected[0..16] { return Err(Error::Auth); } - - let mut header_dec = H128::new(); - self.decoder.decrypt(&mut RefReadBuffer::new(&header[0..16]), &mut RefWriteBuffer::new(&mut header_dec), false).expect("Invalid length or padding"); - - let length = ((header[0] as u32) << 8 + header[1] as u32) << 8 + header[2] as u32; - let header_rlp = UntrustedRlp::new(&header[3..]); + + let mut hdec = H128::new(); + self.decoder.decrypt(&mut RefReadBuffer::new(&header[0..16]), &mut RefWriteBuffer::new(&mut hdec), false).expect("Invalid length or padding"); + + let length = ((((hdec[0] as u32) << 8) + (hdec[1] as u32)) << 8) + (hdec[2] as u32); + let header_rlp = UntrustedRlp::new(&hdec[3..6]); let protocol_id = try!(u16::decode_untrusted(&try!(header_rlp.at(0)))); self.payload_len = length; @@ -263,6 +272,7 @@ impl EncryptedConnection { return Err(Error::Auth); } self.ingress_mac.update(&payload[0..payload.len() - 16]); + EncryptedConnection::update_mac(&mut self.ingress_mac, &mut self.mac_encoder, &[0u8; 0]); let mac = &payload[(payload.len() - 16)..]; let mut expected = H128::new(); self.ingress_mac.clone().finalize(&mut expected); @@ -279,14 +289,32 @@ impl EncryptedConnection { }) } + fn update_mac(mac: &mut Keccak, mac_encoder: &mut EcbEncryptor>, seed: &[u8]) { + let mut prev = H128::new(); + mac.clone().finalize(&mut prev); + let mut enc = H128::new(); + println!("before: {}", prev.hex()); + mac_encoder.encrypt(&mut RefReadBuffer::new(&prev), &mut RefWriteBuffer::new(&mut enc), true).unwrap(); + mac_encoder.reset(); + println!("after {}", enc.hex()); + + if !seed.is_empty() { + enc = enc ^ H128::from_slice(seed); + } + else { + enc = enc ^ prev; + } + mac.update(&enc); + } + pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.reregister(event_loop)); match self.read_state { EncryptedConnectionState::Header => { match try!(self.connection.readable()) { - Some(data) => { - try!(self.read_header(&data)); + Some(data) => { + try!(self.read_header(&data)); }, None => {} }; @@ -294,7 +322,7 @@ impl EncryptedConnection { }, EncryptedConnectionState::Payload => { match try!(self.connection.readable()) { - Some(data) => { + Some(data) => { self.read_state = EncryptedConnectionState::Header; self.connection.expect(ENCRYPTED_HEADER_LEN); Ok(Some(try!(self.read_payload(&data)))) @@ -312,12 +340,36 @@ impl EncryptedConnection { Ok(()) } - pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { self.connection.expect(ENCRYPTED_HEADER_LEN); self.idle_timeout.map(|t| event_loop.clear_timeout(t)); - self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); - try!(self.connection.register(event_loop)); + self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); + try!(self.connection.reregister(event_loop)); Ok(()) - } + } } +#[test] +pub fn ctest() { + use hash::*; + use std::str::FromStr; + let key = H256::from_str("2212767d793a7a3d66f869ae324dd11bd17044b82c9f463b8a541a4d089efec5").unwrap(); + let before = H128::from_str("12532abaec065082a3cf1da7d0136f15").unwrap(); + let before2 = H128::from_str("7e99f682356fdfbc6b67a9562787b18a").unwrap(); + let after = H128::from_str("89464c6b04e7c99e555c81d3f7266a05").unwrap(); + let after2 = H128::from_str("85c070030589ef9c7a2879b3a8489316").unwrap(); + + let mut got = H128::new(); + + let mut encoder = EcbEncryptor::new(AesSafe256Encryptor::new(&key), NoPadding); + encoder.encrypt(&mut RefReadBuffer::new(&before), &mut RefWriteBuffer::new(&mut got), true).unwrap(); + encoder.reset(); + println!("got: {} ", got.hex()); + assert_eq!(got, after); + got = H128::new(); + encoder.encrypt(&mut RefReadBuffer::new(&before2), &mut RefWriteBuffer::new(&mut got), true).unwrap(); + encoder.reset(); + assert_eq!(got, after2); +} + + diff --git a/src/network/discovery.rs b/src/network/discovery.rs new file mode 100644 index 000000000..e8a342c22 --- /dev/null +++ b/src/network/discovery.rs @@ -0,0 +1,204 @@ +// This module is a work in progress + +#![allow(dead_code)] //TODO: remove this after everything is done + +use std::collections::{HashSet, BTreeMap}; +use std::cell::{RefCell}; +use std::ops::{DerefMut}; +use mio::*; +use mio::udp::*; +use hash::*; +use crypto::*; +use network::host::*; + +const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. +const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia]. +const NODE_BINS: u32 = ADDRESS_BITS - 1; ///< Size of m_state (excludes root, which is us). +const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover) +const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. +const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. + +struct NodeBucket { + distance: u32, + nodes: Vec +} + +impl NodeBucket { + fn new(distance: u32) -> NodeBucket { + NodeBucket { + distance: distance, + nodes: Vec::new() + } + } +} + +struct Discovery { + id: NodeId, + discovery_round: u16, + discovery_id: NodeId, + discovery_nodes: HashSet, + node_buckets: Vec, +} + +struct FindNodePacket; + +impl FindNodePacket { + fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { + FindNodePacket + } + fn sign(&mut self, _secret: &Secret) { + } + + fn send(& self, _socket: &mut UdpSocket) { + } +} + +impl Discovery { + pub fn new(id: &NodeId) -> Discovery { + Discovery { + id: id.clone(), + discovery_round: 0, + discovery_id: NodeId::new(), + discovery_nodes: HashSet::new(), + node_buckets: (0..NODE_BINS).map(|x| NodeBucket::new(x)).collect(), + } + } + + pub fn add_node(&mut self, id: &NodeId) { + self.node_buckets[Discovery::distance(&self.id, &id) as usize].nodes.push(id.clone()); + } + + fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { + self.discovery_round = 0; + self.discovery_id.randomize(); + self.discovery_nodes.clear(); + self.discover(event_loop); + } + + fn discover(&mut self, event_loop: &mut EventLoop) { + if self.discovery_round == DISCOVERY_MAX_STEPS + { + debug!("Restarting discovery"); + self.start_node_discovery(event_loop); + return; + } + let mut tried_count = 0; + { + let nearest = Discovery::nearest_node_entries(&self.id, &self.discovery_id, &self.node_buckets).into_iter(); + let nodes = RefCell::new(&mut self.discovery_nodes); + let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); + for r in nearest { + //let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id); + //p.sign(&self.secret); + //p.send(&mut self.udp_socket); + let mut borrowed = nodes.borrow_mut(); + borrowed.deref_mut().insert(r.clone()); + tried_count += 1; + } + } + + if tried_count == 0 + { + debug!("Restarting discovery"); + self.start_node_discovery(event_loop); + return; + } + self.discovery_round += 1; + //event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap(); + } + + fn distance(a: &NodeId, b: &NodeId) -> u32 { + let d = a.sha3() ^ b.sha3(); + let mut ret:u32 = 0; + for i in 0..32 { + let mut v: u8 = d[i]; + while v != 0 { + v >>= 1; + ret += 1; + } + } + ret + } + + fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b Vec) -> Vec<&'b NodeId> + { + // send ALPHA FindNode packets to nodes we know, closest to target + const LAST_BIN: u32 = NODE_BINS - 1; + let mut head = Discovery::distance(source, target); + let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; + + let mut found: BTreeMap> = BTreeMap::new(); + let mut count = 0; + + // if d is 0, then we roll look forward, if last, we reverse, else, spread from d + if head > 1 && tail != LAST_BIN { + while head != tail && head < NODE_BINS && count < BUCKET_SIZE + { + for n in buckets[head as usize].nodes.iter() + { + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + if count < BUCKET_SIZE && tail != 0 { + for n in buckets[tail as usize].nodes.iter() { + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + } + + head += 1; + if tail > 0 { + tail -= 1; + } + } + } + else if head < 2 { + while head < NODE_BINS && count < BUCKET_SIZE { + for n in buckets[head as usize].nodes.iter() { + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + head += 1; + } + } + else { + while tail > 0 && count < BUCKET_SIZE { + for n in buckets[tail as usize].nodes.iter() { + if count < BUCKET_SIZE { + count += 1; + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + tail -= 1; + } + } + + let mut ret:Vec<&NodeId> = Vec::new(); + for (_, nodes) in found { + for n in nodes { + if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ { + ret.push(n); + } + } + } + ret + } +} diff --git a/src/network/handshake.rs b/src/network/handshake.rs index 7d6c49ae3..788b7c1f7 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -56,7 +56,7 @@ impl Handshake { self.originated = originated; if originated { try!(self.write_auth(host)); - } + } else { self.state = HandshakeState::ReadingAuth; self.connection.expect(AUTH_PACKET_SIZE); @@ -73,17 +73,17 @@ impl Handshake { match self.state { HandshakeState::ReadingAuth => { match try!(self.connection.readable()) { - Some(data) => { - try!(self.read_auth(host, &data)); - try!(self.write_ack()); + Some(data) => { + try!(self.read_auth(host, &data)); + try!(self.write_ack()); }, None => {} }; }, HandshakeState::ReadingAck => { match try!(self.connection.readable()) { - Some(data) => { - try!(self.read_ack(host, &data)); + Some(data) => { + try!(self.read_ack(host, &data)); self.state = HandshakeState::StartSession; }, None => {} @@ -91,7 +91,9 @@ impl Handshake { }, _ => { panic!("Unexpected state") } } - try!(self.connection.reregister(event_loop)); + if self.state != HandshakeState::StartSession { + try!(self.connection.reregister(event_loop)); + } Ok(()) } @@ -101,7 +103,7 @@ impl Handshake { HandshakeState::WritingAuth => { match try!(self.connection.writable()) { WriteStatus::Complete => { - self.connection.expect(ACK_PACKET_SIZE); + self.connection.expect(ACK_PACKET_SIZE); self.state = HandshakeState::ReadingAck; }, _ => {} @@ -109,8 +111,8 @@ impl Handshake { }, HandshakeState::WritingAck => { match try!(self.connection.writable()) { - WriteStatus::Complete => { - self.connection.expect(32); + WriteStatus::Complete => { + self.connection.expect(32); self.state = HandshakeState::StartSession; }, _ => {} @@ -118,7 +120,9 @@ impl Handshake { }, _ => { panic!("Unexpected state") } } - try!(self.connection.reregister(event_loop)); + if self.state != HandshakeState::StartSession { + try!(self.connection.reregister(event_loop)); + } Ok(()) } @@ -155,9 +159,8 @@ impl Handshake { assert!(data.len() == ACK_PACKET_SIZE); self.ack_cipher = data.to_vec(); let ack = try!(ecies::decrypt(host.secret(), data)); - let (pubk, nonce) = ack.split_at(65); - self.remote_public.clone_from_slice(pubk); - self.remote_nonce.clone_from_slice(nonce); + self.remote_public.clone_from_slice(&ack[0..64]); + self.remote_nonce.clone_from_slice(&ack[64..(64+32)]); Ok(()) } @@ -171,7 +174,7 @@ impl Handshake { let (hepubk, rest) = rest.split_at_mut(32); let (pubk, rest) = rest.split_at_mut(64); let (nonce, _) = rest.split_at_mut(32); - + // E(remote-pubk, S(ecdhe-random, ecdh-shared-secret^nonce) || H(ecdhe-random-pubk) || pubk || nonce || 0x0) let shared = try!(crypto::ecdh::agree(host.secret(), &self.id)); try!(crypto::ec::sign(self.ecdhe.secret(), &(&shared ^ &self.nonce))).copy_to(sig); diff --git a/src/network/host.rs b/src/network/host.rs index f0465bc15..e35314d88 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -1,10 +1,8 @@ #![allow(dead_code)] //TODO: remove this after everything is done //TODO: remove all unwraps use std::net::{SocketAddr, ToSocketAddrs}; -use std::collections::{HashSet, HashMap, BTreeMap}; +use std::collections::{HashMap}; use std::hash::{Hash, Hasher}; -use std::cell::{RefCell}; -use std::ops::{DerefMut}; use std::str::{FromStr}; use mio::*; use mio::util::{Slab}; @@ -18,18 +16,11 @@ use network::handshake::Handshake; use network::session::Session; use network::Error; -const DEFAULT_PORT: u16 = 30303; +const DEFAULT_PORT: u16 = 30304; -const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. -const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia]. -const NODE_BINS: u32 = ADDRESS_BITS - 1; ///< Size of m_state (excludes root, which is us). -const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover) const MAX_CONNECTIONS: usize = 1024; const IDEAL_PEERS:u32 = 10; -const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. -const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. - pub type NodeId = H512; #[derive(Debug)] @@ -44,8 +35,8 @@ struct NetworkConfiguration { impl NetworkConfiguration { fn new() -> NetworkConfiguration { NetworkConfiguration { - listen_address: SocketAddr::from_str("0.0.0.0:30303").unwrap(), - public_address: SocketAddr::from_str("0.0.0.0:30303").unwrap(), + listen_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), + public_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), no_nat: false, no_discovery: false, pin: false, @@ -54,7 +45,7 @@ impl NetworkConfiguration { } #[derive(Debug)] -struct NodeEndpoint { +pub struct NodeEndpoint { address: SocketAddr, address_str: String, udp_port: u16 @@ -142,33 +133,6 @@ impl Hash for Node { } } -struct NodeBucket { - distance: u32, - nodes: Vec -} - -impl NodeBucket { - fn new(distance: u32) -> NodeBucket { - NodeBucket { - distance: distance, - nodes: Vec::new() - } - } -} - -struct FindNodePacket; - -impl FindNodePacket { - fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { - FindNodePacket - } - fn sign(&mut self, _secret: &Secret) { - } - - fn send(& self, _socket: &mut UdpSocket) { - } -} - // Tokens const TCP_ACCEPT: usize = 1; const IDLE: usize = 3; @@ -190,7 +154,7 @@ pub struct CapabilityInfo { impl Encodable for CapabilityInfo { fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_list(|e| { + encoder.emit_list(|e| { self.protocol.encode(e); self.version.encode(e); }); @@ -241,10 +205,6 @@ pub struct Host { udp_socket: UdpSocket, listener: TcpListener, connections: Slab, - discovery_round: u16, - discovery_id: NodeId, - discovery_nodes: HashSet, - node_buckets: Vec, nodes: HashMap, idle_timeout: Timeout, } @@ -279,7 +239,7 @@ impl Host { let port = config.listen_address.port(); let mut host = Host { - info: HostInfo { + info: HostInfo { keys: KeyPair::create().unwrap(), config: config, nonce: H256::random(), @@ -292,14 +252,11 @@ impl Host { udp_socket: udp_socket, listener: listener, connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), - discovery_round: 0, - discovery_id: NodeId::new(), - discovery_nodes: HashSet::new(), - node_buckets: (0..NODE_BINS).map(|x| NodeBucket::new(x)).collect(), nodes: HashMap::new(), idle_timeout: idle_timeout }; + host.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); @@ -320,146 +277,11 @@ impl Host { match Node::from_str(id) { Err(e) => { warn!("Could not add node: {:?}", e); }, Ok(n) => { - self.node_buckets[Host::distance(self.info.id(), &n.id) as usize].nodes.push(n.id.clone()); - self.nodes.insert(n.id.clone(), n); + self.nodes.insert(n.id.clone(), n); } } } - fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { - self.discovery_round = 0; - self.discovery_id.randomize(); - self.discovery_nodes.clear(); - self.discover(event_loop); - } - - fn discover(&mut self, event_loop: &mut EventLoop) { - if self.discovery_round == DISCOVERY_MAX_STEPS - { - debug!("Restarting discovery"); - self.start_node_discovery(event_loop); - return; - } - let mut tried_count = 0; - { - let nearest = Host::nearest_node_entries(&self.info.id(), &self.discovery_id, &self.node_buckets).into_iter(); - let nodes = RefCell::new(&mut self.discovery_nodes); - let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); - for r in nearest { - //let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id); - //p.sign(&self.secret); - //p.send(&mut self.udp_socket); - let mut borrowed = nodes.borrow_mut(); - borrowed.deref_mut().insert(r.clone()); - tried_count += 1; - } - } - - if tried_count == 0 - { - debug!("Restarting discovery"); - self.start_node_discovery(event_loop); - return; - } - self.discovery_round += 1; - event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap(); - } - - fn distance(a: &NodeId, b: &NodeId) -> u32 { - let d = a.sha3() ^ b.sha3(); - let mut ret:u32 = 0; - for i in 0..32 { - let mut v: u8 = d[i]; - while v != 0 { - v >>= 1; - ret += 1; - } - } - ret - } - - fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b Vec) -> Vec<&'b NodeId> - { - // send ALPHA FindNode packets to nodes we know, closest to target - const LAST_BIN: u32 = NODE_BINS - 1; - let mut head = Host::distance(source, target); - let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; - - let mut found: BTreeMap> = BTreeMap::new(); - let mut count = 0; - - // if d is 0, then we roll look forward, if last, we reverse, else, spread from d - if head > 1 && tail != LAST_BIN { - while head != tail && head < NODE_BINS && count < BUCKET_SIZE - { - for n in buckets[head as usize].nodes.iter() - { - if count < BUCKET_SIZE { - count += 1; - found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - if count < BUCKET_SIZE && tail != 0 { - for n in buckets[tail as usize].nodes.iter() { - if count < BUCKET_SIZE { - count += 1; - found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - } - - head += 1; - if tail > 0 { - tail -= 1; - } - } - } - else if head < 2 { - while head < NODE_BINS && count < BUCKET_SIZE { - for n in buckets[head as usize].nodes.iter() { - if count < BUCKET_SIZE { - count += 1; - found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - head += 1; - } - } - else { - while tail > 0 && count < BUCKET_SIZE { - for n in buckets[tail as usize].nodes.iter() { - if count < BUCKET_SIZE { - count += 1; - found.entry(Host::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - tail -= 1; - } - } - - let mut ret:Vec<&NodeId> = Vec::new(); - for (_, nodes) in found { - for n in nodes { - if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ { - ret.push(n); - } - } - } - ret - } - fn maintain_network(&mut self, event_loop: &mut EventLoop) { self.connect_peers(event_loop); } @@ -482,7 +304,9 @@ impl Host { let mut to_connect: Vec = Vec::new(); let mut req_conn = 0; - for n in self.node_buckets.iter().flat_map(|n| &n.nodes).map(|id| NodeInfo { id: id.clone(), peer_type: self.nodes.get(id).unwrap().peer_type}) { + //TODO: use nodes from discovery here + //for n in self.node_buckets.iter().flat_map(|n| &n.nodes).map(|id| NodeInfo { id: id.clone(), peer_type: self.nodes.get(id).unwrap().peer_type}) { + for n in self.nodes.values().map(|n| NodeInfo { id: n.id.clone(), peer_type: n.peer_type }) { let connected = self.have_session(&n.id) || self.connecting_to(&n.id); let required = n.peer_type == PeerType::Required; if connected && required { @@ -501,7 +325,7 @@ impl Host { req_conn += 1; } } - + if !self.info.config.pin { let pending_count = 0; //TODO: @@ -533,8 +357,8 @@ impl Host { let socket = { let node = self.nodes.get_mut(id).unwrap(); node.last_attempted = Some(::time::now()); - - + + //blog(NetConnect) << "Attempting connection to node" << _p->id << "@" << ep << "from" << id(); match TcpStream::connect(&node.endpoint.address) { Ok(socket) => socket, @@ -547,7 +371,7 @@ impl Host { let nonce = self.info.next_nonce(); match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(token, id, socket, &nonce).expect("Can't create handshake"))) { - Some(token) => { + Some(token) => { match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.start(&self.info, true) @@ -635,7 +459,7 @@ impl Host { match c { ConnectionEntry::Handshake(h) => Session::new(h, event_loop, info) .map(|s| Some(ConnectionEntry::Session(s))) - .unwrap_or_else(|e| { + .unwrap_or_else(|e| { debug!(target: "net", "Session construction error: {:?}", e); None }), diff --git a/src/network/mod.rs b/src/network/mod.rs index 6426d523a..1199ab967 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -3,6 +3,7 @@ mod host; mod connection; mod handshake; mod session; +mod discovery; #[derive(Debug, Copy, Clone)] pub enum DisconnectReason diff --git a/src/network/session.rs b/src/network/session.rs index 1ed2449b5..eb6ca6ced 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -1,5 +1,5 @@ -#![allow(dead_code)] //TODO: remove this after everything is done -//TODO: hello packet timeout +//#![allow(dead_code)] //TODO: remove this after everything is done + use mio::*; use hash::*; use rlp::*; @@ -30,11 +30,10 @@ const PACKET_PEERS: u8 = 0x05; const PACKET_USER: u8 = 0x10; const PACKET_LAST: u8 = 0x7f; -impl Session { +impl Session { pub fn new(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { let id = h.id.clone(); - let mut connection = try!(EncryptedConnection::new(h)); - try!(connection.register(event_loop)); + let connection = try!(EncryptedConnection::new(h)); let mut session = Session { connection: connection, had_hello: false, @@ -47,13 +46,14 @@ impl Session { }; try!(session.write_hello(host)); try!(session.write_ping()); + try!(session.connection.register(event_loop)); Ok(session) } pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { match try!(self.connection.readable(event_loop)) { - Some(data) => { - try!(self.read_packet(data, host)); + Some(data) => { + try!(self.read_packet(data, host)); }, None => {} }; @@ -105,7 +105,7 @@ impl Session { fn read_hello(&mut self, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), Error> { let protocol = try!(u32::decode_untrusted(&try!(rlp.at(0)))); - let client_version = try!(String::decode_untrusted(&try!(rlp.at(0)))); + let client_version = try!(String::decode_untrusted(&try!(rlp.at(1)))); let mut caps: Vec = try!(Decodable::decode_untrusted(&try!(rlp.at(2)))); let id = try!(NodeId::decode_untrusted(&try!(rlp.at(4)))); diff --git a/src/triehash.rs b/src/triehash.rs index 778dc7f17..ceb4c1535 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,5 +1,5 @@ //! Generete trie root. -//! +//! //! This module should be used to generate trie root hash. use std::collections::BTreeMap; @@ -12,13 +12,13 @@ use vector::SharedPrefix; // todo: verify if example for ordered_trie_root is valid /// Generates a trie root hash for a vector of values -/// +/// /// ```rust /// extern crate ethcore_util as util; /// use std::str::FromStr; /// use util::triehash::*; /// use util::hash::*; -/// +/// /// fn main() { /// let v = vec![From::from("doe"), From::from("reindeer")]; /// let root = "e766d5d51b89dc39d981b41bda63248d7abce4f0225eefd023792a540bcffee3"; @@ -50,7 +50,7 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { /// use std::str::FromStr; /// use util::triehash::*; /// use util::hash::*; -/// +/// /// fn main() { /// let v = vec![ /// (From::from("doe"), From::from("reindeer")), @@ -87,9 +87,9 @@ fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { /// Hex-prefix Notation. First nibble has flags: oddness = 2^0 & termination = 2^1. /// /// The "termination marker" and "leaf-node" specifier are completely equivalent. -/// +/// /// Input values are in range `[0, 0xf]`. -/// +/// /// ```markdown /// [0,0,1,2,3,4,5] 0x10012345 // 7 > 4 /// [0,1,2,3,4,5] 0x00012345 // 6 > 4 @@ -102,7 +102,7 @@ fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { /// [0,1,2,3,4,5,T] 0x20012345 // 6 > 4 /// [1,2,3,4,5,T] 0x312345 // 5 > 3 /// [1,2,3,4,T] 0x201234 // 4 > 3 -/// ``` +/// ``` fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { let inlen = nibbles.len(); let oddness_factor = inlen % 2; @@ -121,7 +121,7 @@ fn hex_prefix_encode(nibbles: &[u8], leaf: bool) -> Vec { res.push(first_byte); - let mut offset = oddness_factor; + let mut offset = oddness_factor; while offset < inlen { let byte = (nibbles[offset] << 4) + nibbles[offset + 1]; res.push(byte); @@ -169,7 +169,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre // skip first element .skip(1) // get minimum number of shared nibbles between first and each successive - .fold(key.len(), | acc, &(ref k, _) | { + .fold(key.len(), | acc, &(ref k, _) | { cmp::min(key.shared_prefix_len(&k), acc) }); @@ -184,7 +184,7 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre } // an item for every possible nibble/suffix - // + 1 for data + // + 1 for data stream.append_list(17); // if first key len is equal to prefix_len, move to next element @@ -199,10 +199,10 @@ fn hash256rlp(input: &[(Vec, Vec)], pre_len: usize, stream: &mut RlpStre let len = match begin < input.len() { true => input[begin..].iter() .take_while(| pair | pair.0[pre_len] == i ) - .count(), + .count(), false => 0 }; - + // if at least 1 successive element has the same nibble // append their suffixes match len { @@ -238,7 +238,7 @@ fn test_nibbles() { // A => 65 => 0x41 => [4, 1] let v: Vec = From::from("A"); - let e = vec![4, 1]; + let e = vec![4, 1]; assert_eq!(as_nibbles(&v), e); } @@ -303,7 +303,7 @@ mod tests { (From::from("foo"), From::from("bar")), (From::from("food"), From::from("bass")) ]; - + assert_eq!(trie_root(v), H256::from_str("17beaa1648bafa633cda809c90c04af50fc8aed3cb40d16efbddee6fdf63c4c3").unwrap()); } @@ -315,7 +315,7 @@ mod tests { (From::from("dog"), From::from("puppy")), (From::from("dogglesworth"), From::from("cat")), ]; - + assert_eq!(trie_root(v), H256::from_str("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3").unwrap()); } @@ -328,7 +328,7 @@ mod tests { (From::from("doge"), From::from("coin")), (From::from("horse"), From::from("stallion")), ]; - + assert_eq!(trie_root(v), H256::from_str("5991bb8c6514148a29db676a14ac506cd2cd5775ace63c30a4fe457715e9ac84").unwrap()); } @@ -349,8 +349,8 @@ mod tests { #[test] fn test_trie_root() { let v = vec![ - - ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), + + ("0000000000000000000000000000000000000000000000000000000000000045".from_hex().unwrap(), "22b224a1420a802ab51d326e29fa98e34c4f24ea".from_hex().unwrap()), ("0000000000000000000000000000000000000000000000000000000000000046".from_hex().unwrap(), @@ -381,16 +381,16 @@ mod tests { #[test] fn test_triehash_json_trietest_json() { - let data = include_bytes!("../tests/TrieTests/trietest.json"); + //let data = include_bytes!("../tests/TrieTests/trietest.json"); - let s = String::from_bytes(data).unwrap(); - let json = Json::from_str(&s).unwrap(); - let obj = json.as_object().unwrap(); + //let s = String::from_bytes(data).unwrap(); + //let json = Json::from_str(&s).unwrap(); + //let obj = json.as_object().unwrap(); - for (key, value) in obj.iter() { - println!("running test: {}", key); - } - assert!(false); + //for (key, value) in obj.iter() { + // println!("running test: {}", key); + //} + //assert!(false); } } From 1cb6cc0b87a830e283226c761ed13036d24fc349 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 3 Dec 2015 18:42:51 +0100 Subject: [PATCH 217/379] Stress-tests for trie. All passing. --- src/trie.rs | 135 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 94 insertions(+), 41 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 9b31545db..81ac4f99b 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -801,28 +801,86 @@ mod tests { use rlp; use env_logger; use rand::random; - use bytes::ToPretty; + use std::collections::HashSet; + use bytes::{ToPretty,Bytes}; + + fn random_key() -> Vec { + let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; + let mut ret: Vec = Vec::new(); + let r = random::() % 4 + 1; + for _ in 0..r { + ret.push(chars[random::() % chars.len()]); + } + ret + } + + fn random_value(i: usize) -> Bytes { + match random::() % 2 { + 0 => rlp::encode(&i), + _ => { + let mut h = H256::new(); + h.mut_bytes()[31] = i as u8; + rlp::encode(&h) + }, + } + } + + fn populate_trie(v: &Vec<(Vec, Vec)>) -> TrieDB { + let mut t = TrieDB::new_memory(); + for i in 0..v.len() { + let key: &[u8]= &v[i].0; + let val: &[u8] = &v[i].1; + t.insert(&key, &val); + } + t + } + + fn unpopulate_trie(t: &mut TrieDB, v: &Vec<(Vec, Vec)>) { + for i in v.iter() { + let key: &[u8]= &i.0; + t.remove(&key); + } + } #[test] fn playpen() { env_logger::init().ok(); + for _ in 0..1000 { + let mut x: Vec<(Vec, Vec)> = Vec::new(); + let mut got: HashSet> = HashSet::new(); + for j in 0..10usize { + let key = random_key(); + if !got.contains(&key) { + x.push((key.clone(), random_value(j))); + got.insert(key); + } + } - let big_value = b"00000000000000000000000000000000"; - - let mut t1 = TrieDB::new_memory(); - t1.insert(&[0x01, 0x23], &big_value.to_vec()); - t1.insert(&[0x01, 0x34], &big_value.to_vec()); - trace!("keys remaining {:?}", t1.db_items_remaining()); - assert!(t1.db_items_remaining().is_empty()); - let mut t2 = TrieDB::new_memory(); - t2.insert(&[0x01], &big_value.to_vec()); - t2.insert(&[0x01, 0x23], &big_value.to_vec()); - t2.insert(&[0x01, 0x34], &big_value.to_vec()); - t2.remove(&[0x01]); - assert!(t2.db_items_remaining().is_empty()); - /*if t1.root() != t2.root()*/ { - trace!("{:?}", t1); - trace!("{:?}", t2); + let real = trie_root(x.clone()); + let mut memtrie = populate_trie(&x); + if *memtrie.root() != real || !memtrie.db_items_remaining().is_empty() { + println!("TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in x.iter() { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("{:?}", memtrie); + } + assert_eq!(*memtrie.root(), real); + assert!(memtrie.db_items_remaining().is_empty()); + unpopulate_trie(&mut memtrie, &x); + if *memtrie.root() != SHA3_NULL_RLP || !memtrie.db_items_remaining().is_empty() { + println!("TRIE MISMATCH"); + println!(""); + println!("{:?} vs {:?}", memtrie.root(), real); + for i in x.iter() { + println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); + } + println!("{:?}", memtrie); + } + assert_eq!(*memtrie.root(), SHA3_NULL_RLP); + assert!(memtrie.db_items_remaining().is_empty()); } } @@ -842,6 +900,23 @@ mod tests { #[test] fn remove_to_empty() { + let big_value = b"00000000000000000000000000000000"; + + let mut t1 = TrieDB::new_memory(); + t1.insert(&[0x01, 0x23], &big_value.to_vec()); + t1.insert(&[0x01, 0x34], &big_value.to_vec()); + trace!("keys remaining {:?}", t1.db_items_remaining()); + assert!(t1.db_items_remaining().is_empty()); + let mut t2 = TrieDB::new_memory(); + t2.insert(&[0x01], &big_value.to_vec()); + t2.insert(&[0x01, 0x23], &big_value.to_vec()); + t2.insert(&[0x01, 0x34], &big_value.to_vec()); + t2.remove(&[0x01]); + assert!(t2.db_items_remaining().is_empty()); + /*if t1.root() != t2.root()*/ { + trace!("{:?}", t1); + trace!("{:?}", t2); + } } #[test] @@ -1022,16 +1097,6 @@ mod tests { //assert!(false); } - fn random_key() -> Vec { - let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; - let mut ret: Vec = Vec::new(); - let r = random::() % 4 + 1; - for _ in 0..r { - ret.push(chars[random::() % chars.len()]); - } - ret - } - #[test] fn stress() { for _ in 0..5000 { @@ -1041,10 +1106,10 @@ mod tests { x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); - let memtrie = trie_root_mem(&x); + let memtrie = populate_trie(&x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let memtrie_sorted = trie_root_mem(&y); + let memtrie_sorted = populate_trie(&y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1064,18 +1129,6 @@ mod tests { } } - fn trie_root_mem(v: &Vec<(Vec, Vec)>) -> TrieDB { - let mut t = TrieDB::new_memory(); - - for i in 0..v.len() { - let key: &[u8]= &v[i].0; - let val: &[u8] = &v[i].1; - t.insert(&key, &val); - } - - t - } - #[test] fn test_trie_json() { println!("Json trie test: "); From 668d735fa7bddd56252e3a9940aac6f6594c5a22 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 3 Dec 2015 19:07:08 +0100 Subject: [PATCH 218/379] U256 to byts conversion --- src/hash.rs | 15 +++++++++++++-- src/uint.rs | 29 ++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index ad918ef3d..5662d713c 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -9,6 +9,7 @@ use rand::Rng; use rand::os::OsRng; use bytes::BytesConvertable; use math::log2; +use uint::U256; /// types implementing FixedHash must be also BytesConvertable pub trait FixedHash: Sized + BytesConvertable { @@ -307,6 +308,16 @@ macro_rules! impl_hash { } } +impl<'a> From<&'a U256> for H256 { + fn from(value: &'a U256) -> H256 { + unsafe { + let mut ret: H256 = ::std::mem::uninitialized(); + value.to_bytes(&mut ret); + ret + } + } +} + impl_hash!(H32, 4); impl_hash!(H64, 8); impl_hash!(H128, 16); @@ -350,7 +361,7 @@ mod tests { #[test] fn shift_bloom() { use sha3::Hashable; - + let bloom = H2048::from_str("00000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002020000000000000000000000000000000000000000000008000000001000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); let address = Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(); let topic = H256::from_str("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); @@ -362,7 +373,7 @@ mod tests { my_bloom.shift_bloom(&address.sha3()); assert!(my_bloom.contains_bloom(&address.sha3())); assert!(!my_bloom.contains_bloom(&topic.sha3())); - + my_bloom.shift_bloom(&topic.sha3()); assert_eq!(my_bloom, bloom); assert!(my_bloom.contains_bloom(&address.sha3())); diff --git a/src/uint.rs b/src/uint.rs index c8e9d84de..693b9f471 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -186,6 +186,15 @@ macro_rules! construct_uint { (arr[index / 8] >> ((index % 8)) * 8) as u8 } + pub fn to_bytes(&self, bytes: &mut[u8]) { + assert!($n_words * 8 == bytes.len()); + let &$name(ref arr) = self; + for i in 0..bytes.len() { + let rev = bytes.len() - 1 - i; + let pos = rev / 8; + bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; + } + } /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -287,7 +296,7 @@ macro_rules! construct_uint { let mut sub_copy = self; let mut shift_copy = other; let mut ret = [0u64; $n_words]; - + let my_bits = self.bits(); let your_bits = other.bits(); @@ -453,7 +462,7 @@ construct_uint!(U128, 2); impl From for U256 { fn from(value: U128) -> U256 { - let U128(ref arr) = value; + let U128(ref arr) = value; let mut ret = [0; 4]; ret[0] = arr[0]; ret[1] = arr[1]; @@ -483,7 +492,7 @@ mod tests { assert_eq!(e, ub); assert_eq!(e, uc); assert_eq!(e, ud); - + // test initialization from bytes let va = U256::from(&[10u8][..]); assert_eq!(e, va); @@ -497,7 +506,7 @@ mod tests { assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from(&[ 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); assert_eq!(U256([0x00192437100019fa, 0x243710, 0, 0]), U256::from(&[ 0x24u8, 0x37, 0x10, @@ -514,6 +523,16 @@ mod tests { assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); } + #[test] + pub fn uint256_to() { + let hex = "8090a0b0c0d0e0f00910203040506077583a2cf8264910e1436bda32571012f0"; + let uint = U256::from_str(hex).unwrap(); + let mut bytes = [0u8; 32]; + uint.to_bytes(&mut bytes); + let uint2 = U256::from(&bytes[..]); + assert_eq!(uint, uint2); + } + #[test] pub fn uint256_bits_test() { assert_eq!(U256::from(0u64).bits(), 0); @@ -522,7 +541,7 @@ mod tests { assert_eq!(U256::from(300u64).bits(), 9); assert_eq!(U256::from(60000u64).bits(), 16); assert_eq!(U256::from(70000u64).bits(), 17); - + //// Try to read the following lines out loud quickly let mut shl = U256::from(70000u64); shl = shl << 100; From 40f5bb6da27384e954e3c74fa473b3af63234c47 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 03:02:53 +0100 Subject: [PATCH 219/379] cleanedup uint.rs, converted spaces to tabs --- src/uint.rs | 937 +++++++++++++++++++++++----------------------------- 1 file changed, 412 insertions(+), 525 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index 693b9f471..e017de385 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -22,452 +22,339 @@ ///! use std::fmt; -use std::cmp::{Ord, PartialOrd, Ordering}; +use std::cmp::*; use std::ops::*; use std::str::FromStr; use rustc_serialize::hex::{FromHex, FromHexError}; macro_rules! impl_map_from { - ($thing:ident, $from:ty, $to:ty) => { - impl From<$from> for $thing { - fn from(value: $from) -> $thing { - From::from(value as $to) - } - } - } -} - -macro_rules! impl_array_newtype { - ($thing:ident, $ty:ty, $len:expr) => { - impl $thing { - #[inline] - /// Converts the object to a raw pointer - pub fn as_ptr(&self) -> *const $ty { - let &$thing(ref dat) = self; - dat.as_ptr() - } - - #[inline] - /// Converts the object to a mutable raw pointer - pub fn as_mut_ptr(&mut self) -> *mut $ty { - let &mut $thing(ref mut dat) = self; - dat.as_mut_ptr() - } - - #[inline] - /// Returns the length of the object as an array - pub fn len(&self) -> usize { $len } - - #[inline] - /// Returns whether the object, as an array, is empty. Always false. - pub fn is_empty(&self) -> bool { false } - } - - impl<'a> From<&'a [$ty]> for $thing { - fn from(data: &'a [$ty]) -> $thing { - assert_eq!(data.len(), $len); - unsafe { - use std::intrinsics::copy_nonoverlapping; - use std::mem; - let mut ret: $thing = mem::uninitialized(); - copy_nonoverlapping(data.as_ptr(), - ret.as_mut_ptr(), - mem::size_of::<$thing>()); - ret - } - } - } - - impl Index for $thing { - type Output = $ty; - - #[inline] - fn index(&self, index: usize) -> &$ty { - let &$thing(ref dat) = self; - &dat[index] - } - } - - impl_index_newtype!($thing, $ty); - - impl PartialEq for $thing { - #[inline] - fn eq(&self, other: &$thing) -> bool { - &self[..] == &other[..] - } - } - - impl Eq for $thing {} - - impl Clone for $thing { - #[inline] - fn clone(&self) -> $thing { - $thing::from(&self[..]) - } - } - - impl Copy for $thing {} - } -} - -macro_rules! impl_index_newtype { - ($thing:ident, $ty:ty) => { - impl Index> for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, index: Range) -> &[$ty] { - &self.0[index] - } - } - - impl Index> for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, index: RangeTo) -> &[$ty] { - &self.0[index] - } - } - - impl Index> for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, index: RangeFrom) -> &[$ty] { - &self.0[index] - } - } - - impl Index for $thing { - type Output = [$ty]; - - #[inline] - fn index(&self, _: RangeFull) -> &[$ty] { - &self.0[..] - } - } - } + ($thing:ident, $from:ty, $to:ty) => { + impl From<$from> for $thing { + fn from(value: $from) -> $thing { + From::from(value as $to) + } + } + } } macro_rules! construct_uint { - ($name:ident, $n_words:expr) => ( - /// Little-endian large integer type - pub struct $name(pub [u64; $n_words]); - impl_array_newtype!($name, u64, $n_words); + ($name:ident, $n_words:expr) => ( + /// Little-endian large integer type + #[derive(Copy, Clone, Eq, PartialEq)] + pub struct $name(pub [u64; $n_words]); - impl $name { - /// Conversion to u32 - #[inline] - fn low_u32(&self) -> u32 { - let &$name(ref arr) = self; - arr[0] as u32 - } + impl $name { + /// Conversion to u32 + #[inline] + fn low_u32(&self) -> u32 { + let &$name(ref arr) = self; + arr[0] as u32 + } - /// Return the least number of bits needed to represent the number - #[inline] - pub fn bits(&self) -> usize { - let &$name(ref arr) = self; - for i in 1..$n_words { - if arr[$n_words - i] > 0 { return (0x40 * ($n_words - i + 1)) - arr[$n_words - i].leading_zeros() as usize; } - } - 0x40 - arr[0].leading_zeros() as usize - } + /// Return the least number of bits needed to represent the number + #[inline] + pub fn bits(&self) -> usize { + let &$name(ref arr) = self; + for i in 1..$n_words { + if arr[$n_words - i] > 0 { return (0x40 * ($n_words - i + 1)) - arr[$n_words - i].leading_zeros() as usize; } + } + 0x40 - arr[0].leading_zeros() as usize + } - #[inline] - pub fn bit(&self, index: usize) -> bool { - let &$name(ref arr) = self; - arr[index / 64] & (1 << (index % 64)) != 0 - } + #[inline] + pub fn bit(&self, index: usize) -> bool { + let &$name(ref arr) = self; + arr[index / 64] & (1 << (index % 64)) != 0 + } - #[inline] - pub fn byte(&self, index: usize) -> u8 { - let &$name(ref arr) = self; - (arr[index / 8] >> ((index % 8)) * 8) as u8 - } + #[inline] + pub fn byte(&self, index: usize) -> u8 { + let &$name(ref arr) = self; + (arr[index / 8] >> ((index % 8)) * 8) as u8 + } - pub fn to_bytes(&self, bytes: &mut[u8]) { - assert!($n_words * 8 == bytes.len()); - let &$name(ref arr) = self; - for i in 0..bytes.len() { - let rev = bytes.len() - 1 - i; - let pos = rev / 8; - bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; - } - } - /// Multiplication by u32 - fn mul_u32(self, other: u32) -> $name { - let $name(ref arr) = self; - let mut carry = [0u64; $n_words]; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - let upper = other as u64 * (arr[i] >> 32); - let lower = other as u64 * (arr[i] & 0xFFFFFFFF); - if i < 3 { - carry[i + 1] += upper >> 32; - } - ret[i] = lower + (upper << 32); - } - $name(ret) + $name(carry) - } - } + pub fn to_bytes(&self, bytes: &mut[u8]) { + assert!($n_words * 8 == bytes.len()); + let &$name(ref arr) = self; + for i in 0..bytes.len() { + let rev = bytes.len() - 1 - i; + let pos = rev / 8; + bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; + } + } + /// Multiplication by u32 + fn mul_u32(self, other: u32) -> $name { + let $name(ref arr) = self; + let mut carry = [0u64; $n_words]; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + let upper = other as u64 * (arr[i] >> 32); + let lower = other as u64 * (arr[i] & 0xFFFFFFFF); + if i < 3 { + carry[i + 1] += upper >> 32; + } + ret[i] = lower + (upper << 32); + } + $name(ret) + $name(carry) + } + } - impl From for $name { - fn from(value: u64) -> $name { - let mut ret = [0; $n_words]; - ret[0] = value; - $name(ret) - } - } + impl From for $name { + fn from(value: u64) -> $name { + let mut ret = [0; $n_words]; + ret[0] = value; + $name(ret) + } + } - impl_map_from!($name, u8, u64); - impl_map_from!($name, u16, u64); - impl_map_from!($name, u32, u64); + impl_map_from!($name, u8, u64); + impl_map_from!($name, u16, u64); + impl_map_from!($name, u32, u64); - impl<'a> From<&'a [u8]> for $name { - fn from(bytes: &[u8]) -> $name { - assert!($n_words * 8 >= bytes.len()); + impl<'a> From<&'a [u8]> for $name { + fn from(bytes: &[u8]) -> $name { + assert!($n_words * 8 >= bytes.len()); - let mut ret = [0; $n_words]; - for i in 0..bytes.len() { - let rev = bytes.len() - 1 - i; - let pos = rev / 8; - ret[pos] += (bytes[i] as u64) << (rev % 8) * 8; - } - $name(ret) - } - } + let mut ret = [0; $n_words]; + for i in 0..bytes.len() { + let rev = bytes.len() - 1 - i; + let pos = rev / 8; + ret[pos] += (bytes[i] as u64) << (rev % 8) * 8; + } + $name(ret) + } + } - impl FromStr for $name { - type Err = FromHexError; + impl FromStr for $name { + type Err = FromHexError; - fn from_str(value: &str) -> Result<$name, Self::Err> { - let bytes: &[u8] = &try!(value.from_hex()); - Ok(From::from(bytes)) - } - } + fn from_str(value: &str) -> Result<$name, Self::Err> { + let bytes: &[u8] = &try!(value.from_hex()); + Ok(From::from(bytes)) + } + } - impl Add<$name> for $name { - type Output = $name; + impl Add<$name> for $name { + type Output = $name; - fn add(self, other: $name) -> $name { - let $name(ref me) = self; - let $name(ref you) = other; - let mut ret = [0u64; $n_words]; - let mut carry = [0u64; $n_words]; - let mut b_carry = false; - for i in 0..$n_words { - ret[i] = me[i].wrapping_add(you[i]); - if i < $n_words - 1 && ret[i] < me[i] { - carry[i + 1] = 1; - b_carry = true; - } - } - if b_carry { $name(ret) + $name(carry) } else { $name(ret) } - } - } + fn add(self, other: $name) -> $name { + let $name(ref me) = self; + let $name(ref you) = other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + if i < $n_words - 1 && ret[i] < me[i] { + carry[i + 1] = 1; + b_carry = true; + } + } + if b_carry { $name(ret) + $name(carry) } else { $name(ret) } + } + } - impl Sub<$name> for $name { - type Output = $name; + impl Sub<$name> for $name { + type Output = $name; - #[inline] - fn sub(self, other: $name) -> $name { - self + !other + From::from(1u64) - } - } + #[inline] + fn sub(self, other: $name) -> $name { + self + !other + From::from(1u64) + } + } - impl Mul<$name> for $name { - type Output = $name; + impl Mul<$name> for $name { + type Output = $name; - fn mul(self, other: $name) -> $name { - let mut me = self; - // TODO: be more efficient about this - for i in 0..(2 * $n_words) { - me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); - } - me - } - } + fn mul(self, other: $name) -> $name { + let mut me = self; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); + } + me + } + } - impl Div<$name> for $name { - type Output = $name; + impl Div<$name> for $name { + type Output = $name; - fn div(self, other: $name) -> $name { - let mut sub_copy = self; - let mut shift_copy = other; - let mut ret = [0u64; $n_words]; + fn div(self, other: $name) -> $name { + let mut sub_copy = self; + let mut shift_copy = other; + let mut ret = [0u64; $n_words]; - let my_bits = self.bits(); - let your_bits = other.bits(); + let my_bits = self.bits(); + let your_bits = other.bits(); - // Check for division by 0 - assert!(your_bits != 0); + // Check for division by 0 + assert!(your_bits != 0); - // Early return in case we are dividing by a larger number than us - if my_bits < your_bits { - return $name(ret); - } + // Early return in case we are dividing by a larger number than us + if my_bits < your_bits { + return $name(ret); + } - // Bitwise long division - let mut shift = my_bits - your_bits; - shift_copy = shift_copy << shift; - loop { - if sub_copy >= shift_copy { - ret[shift / 64] |= 1 << (shift % 64); - sub_copy = sub_copy - shift_copy; - } - shift_copy = shift_copy >> 1; - if shift == 0 { break; } - shift -= 1; - } + // Bitwise long division + let mut shift = my_bits - your_bits; + shift_copy = shift_copy << shift; + loop { + if sub_copy >= shift_copy { + ret[shift / 64] |= 1 << (shift % 64); + sub_copy = sub_copy - shift_copy; + } + shift_copy = shift_copy >> 1; + if shift == 0 { break; } + shift -= 1; + } - $name(ret) - } - } + $name(ret) + } + } - impl BitAnd<$name> for $name { - type Output = $name; + impl BitAnd<$name> for $name { + type Output = $name; - #[inline] - fn bitand(self, other: $name) -> $name { - let $name(ref arr1) = self; - let $name(ref arr2) = other; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = arr1[i] & arr2[i]; - } - $name(ret) - } - } + #[inline] + fn bitand(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] & arr2[i]; + } + $name(ret) + } + } - impl BitXor<$name> for $name { - type Output = $name; + impl BitXor<$name> for $name { + type Output = $name; - #[inline] - fn bitxor(self, other: $name) -> $name { - let $name(ref arr1) = self; - let $name(ref arr2) = other; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = arr1[i] ^ arr2[i]; - } - $name(ret) - } - } + #[inline] + fn bitxor(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] ^ arr2[i]; + } + $name(ret) + } + } - impl BitOr<$name> for $name { - type Output = $name; + impl BitOr<$name> for $name { + type Output = $name; - #[inline] - fn bitor(self, other: $name) -> $name { - let $name(ref arr1) = self; - let $name(ref arr2) = other; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = arr1[i] | arr2[i]; - } - $name(ret) - } - } + #[inline] + fn bitor(self, other: $name) -> $name { + let $name(ref arr1) = self; + let $name(ref arr2) = other; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = arr1[i] | arr2[i]; + } + $name(ret) + } + } - impl Not for $name { - type Output = $name; + impl Not for $name { + type Output = $name; - #[inline] - fn not(self) -> $name { - let $name(ref arr) = self; - let mut ret = [0u64; $n_words]; - for i in 0..$n_words { - ret[i] = !arr[i]; - } - $name(ret) - } - } + #[inline] + fn not(self) -> $name { + let $name(ref arr) = self; + let mut ret = [0u64; $n_words]; + for i in 0..$n_words { + ret[i] = !arr[i]; + } + $name(ret) + } + } - impl Shl for $name { - type Output = $name; + impl Shl for $name { + type Output = $name; - fn shl(self, shift: usize) -> $name { - let $name(ref original) = self; - let mut ret = [0u64; $n_words]; - let word_shift = shift / 64; - let bit_shift = shift % 64; - for i in 0..$n_words { - // Shift - if bit_shift < 64 && i + word_shift < $n_words { - ret[i + word_shift] += original[i] << bit_shift; - } - // Carry - if bit_shift > 0 && i + word_shift + 1 < $n_words { - ret[i + word_shift + 1] += original[i] >> (64 - bit_shift); - } - } - $name(ret) - } - } + fn shl(self, shift: usize) -> $name { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in 0..$n_words { + // Shift + if bit_shift < 64 && i + word_shift < $n_words { + ret[i + word_shift] += original[i] << bit_shift; + } + // Carry + if bit_shift > 0 && i + word_shift + 1 < $n_words { + ret[i + word_shift + 1] += original[i] >> (64 - bit_shift); + } + } + $name(ret) + } + } - impl Shr for $name { - type Output = $name; + impl Shr for $name { + type Output = $name; - fn shr(self, shift: usize) -> $name { - let $name(ref original) = self; - let mut ret = [0u64; $n_words]; - let word_shift = shift / 64; - let bit_shift = shift % 64; - for i in word_shift..$n_words { - // Shift - ret[i - word_shift] += original[i] >> bit_shift; - // Carry - if bit_shift > 0 && i < $n_words - 1 { - ret[i - word_shift] += original[i + 1] << (64 - bit_shift); - } - } - $name(ret) - } - } + fn shr(self, shift: usize) -> $name { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in word_shift..$n_words { + // Shift + ret[i - word_shift] += original[i] >> bit_shift; + // Carry + if bit_shift > 0 && i < $n_words - 1 { + ret[i - word_shift] += original[i + 1] << (64 - bit_shift); + } + } + $name(ret) + } + } - impl Ord for $name { - fn cmp(&self, other: &$name) -> Ordering { - let &$name(ref me) = self; - let &$name(ref you) = other; - for i in 0..$n_words { - if me[$n_words - 1 - i] < you[$n_words - 1 - i] { return Ordering::Less; } - if me[$n_words - 1 - i] > you[$n_words - 1 - i] { return Ordering::Greater; } - } - Ordering::Equal - } - } + impl Ord for $name { + fn cmp(&self, other: &$name) -> Ordering { + let &$name(ref me) = self; + let &$name(ref you) = other; + for i in 0..$n_words { + if me[$n_words - 1 - i] < you[$n_words - 1 - i] { return Ordering::Less; } + if me[$n_words - 1 - i] > you[$n_words - 1 - i] { return Ordering::Greater; } + } + Ordering::Equal + } + } - impl PartialOrd for $name { - fn partial_cmp(&self, other: &$name) -> Option { - Some(self.cmp(other)) - } - } + impl PartialOrd for $name { + fn partial_cmp(&self, other: &$name) -> Option { + Some(self.cmp(other)) + } + } - impl fmt::Debug for $name { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let &$name(ref data) = self; - try!(write!(f, "0x")); - for ch in data.iter().rev() { - try!(write!(f, "{:02x}", ch)); - } - Ok(()) - } - } - ); + impl fmt::Debug for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let &$name(ref data) = self; + try!(write!(f, "0x")); + for ch in data.iter().rev() { + try!(write!(f, "{:02x}", ch)); + } + Ok(()) + } + } + ); } construct_uint!(U256, 4); construct_uint!(U128, 2); impl From for U256 { - fn from(value: U128) -> U256 { - let U128(ref arr) = value; - let mut ret = [0; 4]; - ret[0] = arr[0]; - ret[1] = arr[1]; - U256(ret) - } + fn from(value: U128) -> U256 { + let U128(ref arr) = value; + let mut ret = [0; 4]; + ret[0] = arr[0]; + ret[1] = arr[1]; + U256(ret) + } } pub const ZERO_U256: U256 = U256([0x00u64; 4]); @@ -476,158 +363,158 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); #[cfg(test)] mod tests { - use uint::U256; - use std::str::FromStr; + use uint::U256; + use std::str::FromStr; - #[test] - pub fn uint256_from() { - let e = U256([10, 0, 0, 0]); + #[test] + pub fn uint256_from() { + let e = U256([10, 0, 0, 0]); - // test unsigned initialization - let ua = U256::from(10u8); - let ub = U256::from(10u16); - let uc = U256::from(10u32); - let ud = U256::from(10u64); - assert_eq!(e, ua); - assert_eq!(e, ub); - assert_eq!(e, uc); - assert_eq!(e, ud); + // test unsigned initialization + let ua = U256::from(10u8); + let ub = U256::from(10u16); + let uc = U256::from(10u32); + let ud = U256::from(10u64); + assert_eq!(e, ua); + assert_eq!(e, ub); + assert_eq!(e, uc); + assert_eq!(e, ud); - // test initialization from bytes - let va = U256::from(&[10u8][..]); - assert_eq!(e, va); + // test initialization from bytes + let va = U256::from(&[10u8][..]); + assert_eq!(e, va); - // more tests for initialization from bytes - assert_eq!(U256([0x1010, 0, 0, 0]), U256::from(&[0x10u8, 0x10][..])); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from(&[0, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from(&[1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from(&[ - 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, - 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); - assert_eq!(U256([0x00192437100019fa, 0x243710, 0, 0]), U256::from(&[ - 0x24u8, 0x37, 0x10, - 0, 0x19, 0x24, 0x37, 0x10, 0, 0x19, 0xfa][..])); + // more tests for initialization from bytes + assert_eq!(U256([0x1010, 0, 0, 0]), U256::from(&[0x10u8, 0x10][..])); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from(&[0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from(&[0, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from(&[1, 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from(&[ + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0x12u8, 0xf0][..])); + assert_eq!(U256([0x00192437100019fa, 0x243710, 0, 0]), U256::from(&[ + 0x24u8, 0x37, 0x10, + 0, 0x19, 0x24, 0x37, 0x10, 0, 0x19, 0xfa][..])); - // test initializtion from string - let sa = U256::from_str("0a").unwrap(); - assert_eq!(e, sa); - assert_eq!(U256([0x1010, 0, 0, 0]), U256::from_str("1010").unwrap()); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); - assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); - assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from_str("0000000012f0").unwrap()); - assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from_str("0100000000000012f0").unwrap()); - assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); - } + // test initializtion from string + let sa = U256::from_str("0a").unwrap(); + assert_eq!(e, sa); + assert_eq!(U256([0x1010, 0, 0, 0]), U256::from_str("1010").unwrap()); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); + assert_eq!(U256([0x12f0, 0, 0, 0]), U256::from_str("12f0").unwrap()); + assert_eq!(U256([0x12f0, 0 , 0, 0]), U256::from_str("0000000012f0").unwrap()); + assert_eq!(U256([0x12f0, 1 , 0, 0]), U256::from_str("0100000000000012f0").unwrap()); + assert_eq!(U256([0x12f0, 1 , 0x0910203040506077, 0x8090a0b0c0d0e0f0]), U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000000100000000000012f0").unwrap()); + } - #[test] - pub fn uint256_to() { + #[test] + pub fn uint256_to() { let hex = "8090a0b0c0d0e0f00910203040506077583a2cf8264910e1436bda32571012f0"; let uint = U256::from_str(hex).unwrap(); let mut bytes = [0u8; 32]; uint.to_bytes(&mut bytes); let uint2 = U256::from(&bytes[..]); - assert_eq!(uint, uint2); - } + assert_eq!(uint, uint2); + } - #[test] - pub fn uint256_bits_test() { - assert_eq!(U256::from(0u64).bits(), 0); - assert_eq!(U256::from(255u64).bits(), 8); - assert_eq!(U256::from(256u64).bits(), 9); - assert_eq!(U256::from(300u64).bits(), 9); - assert_eq!(U256::from(60000u64).bits(), 16); - assert_eq!(U256::from(70000u64).bits(), 17); + #[test] + pub fn uint256_bits_test() { + assert_eq!(U256::from(0u64).bits(), 0); + assert_eq!(U256::from(255u64).bits(), 8); + assert_eq!(U256::from(256u64).bits(), 9); + assert_eq!(U256::from(300u64).bits(), 9); + assert_eq!(U256::from(60000u64).bits(), 16); + assert_eq!(U256::from(70000u64).bits(), 17); - //// Try to read the following lines out loud quickly - let mut shl = U256::from(70000u64); - shl = shl << 100; - assert_eq!(shl.bits(), 117); - shl = shl << 100; - assert_eq!(shl.bits(), 217); - shl = shl << 100; - assert_eq!(shl.bits(), 0); + //// Try to read the following lines out loud quickly + let mut shl = U256::from(70000u64); + shl = shl << 100; + assert_eq!(shl.bits(), 117); + shl = shl << 100; + assert_eq!(shl.bits(), 217); + shl = shl << 100; + assert_eq!(shl.bits(), 0); - //// Bit set check - //// 01010 - assert!(!U256::from(10u8).bit(0)); - assert!(U256::from(10u8).bit(1)); - assert!(!U256::from(10u8).bit(2)); - assert!(U256::from(10u8).bit(3)); - assert!(!U256::from(10u8).bit(4)); + //// Bit set check + //// 01010 + assert!(!U256::from(10u8).bit(0)); + assert!(U256::from(10u8).bit(1)); + assert!(!U256::from(10u8).bit(2)); + assert!(U256::from(10u8).bit(3)); + assert!(!U256::from(10u8).bit(4)); - //// byte check - assert_eq!(U256::from(10u8).byte(0), 10); - assert_eq!(U256::from(0xffu64).byte(0), 0xff); - assert_eq!(U256::from(0xffu64).byte(1), 0); - assert_eq!(U256::from(0x01ffu64).byte(0), 0xff); - assert_eq!(U256::from(0x01ffu64).byte(1), 0x1); - assert_eq!(U256([0u64, 0xfc, 0, 0]).byte(8), 0xfc); - assert_eq!(U256([0u64, 0, 0, u64::max_value()]).byte(31), 0xff); - assert_eq!(U256([0u64, 0, 0, (u64::max_value() >> 8) + 1]).byte(31), 0x01); - } + //// byte check + assert_eq!(U256::from(10u8).byte(0), 10); + assert_eq!(U256::from(0xffu64).byte(0), 0xff); + assert_eq!(U256::from(0xffu64).byte(1), 0); + assert_eq!(U256::from(0x01ffu64).byte(0), 0xff); + assert_eq!(U256::from(0x01ffu64).byte(1), 0x1); + assert_eq!(U256([0u64, 0xfc, 0, 0]).byte(8), 0xfc); + assert_eq!(U256([0u64, 0, 0, u64::max_value()]).byte(31), 0xff); + assert_eq!(U256([0u64, 0, 0, (u64::max_value() >> 8) + 1]).byte(31), 0x01); + } - #[test] - pub fn uint256_comp_test() { - let small = U256([10u64, 0, 0, 0]); - let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); - let bigger = U256([0x9C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); - let biggest = U256([0x5C8C3EE70C644118u64, 0x0209E7378231E632, 0, 1]); + #[test] + pub fn uint256_comp_test() { + let small = U256([10u64, 0, 0, 0]); + let big = U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); + let bigger = U256([0x9C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0]); + let biggest = U256([0x5C8C3EE70C644118u64, 0x0209E7378231E632, 0, 1]); - assert!(small < big); - assert!(big < bigger); - assert!(bigger < biggest); - assert!(bigger <= biggest); - assert!(biggest <= biggest); - assert!(bigger >= big); - assert!(bigger >= small); - assert!(small <= small); - } + assert!(small < big); + assert!(big < bigger); + assert!(bigger < biggest); + assert!(bigger <= biggest); + assert!(biggest <= biggest); + assert!(bigger >= big); + assert!(bigger >= small); + assert!(small <= small); + } - #[test] - pub fn uint256_arithmetic_test() { - let init = U256::from(0xDEADBEEFDEADBEEFu64); - let copy = init; + #[test] + pub fn uint256_arithmetic_test() { + let init = U256::from(0xDEADBEEFDEADBEEFu64); + let copy = init; - let add = init + copy; - assert_eq!(add, U256([0xBD5B7DDFBD5B7DDEu64, 1, 0, 0])); - // Bitshifts - let shl = add << 88; - assert_eq!(shl, U256([0u64, 0xDFBD5B7DDE000000, 0x1BD5B7D, 0])); - let shr = shl >> 40; - assert_eq!(shr, U256([0x7DDE000000000000u64, 0x0001BD5B7DDFBD5B, 0, 0])); - // Increment - let incr = shr + U256::from(1u64); - assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); - // Subtraction - let sub = incr - init; - assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); - // Multiplication - let mult = sub.mul_u32(300); - assert_eq!(mult, U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0])); - // Division - assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8)); - let div = mult / U256::from(300u16); - assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); - //// TODO: bit inversion - } + let add = init + copy; + assert_eq!(add, U256([0xBD5B7DDFBD5B7DDEu64, 1, 0, 0])); + // Bitshifts + let shl = add << 88; + assert_eq!(shl, U256([0u64, 0xDFBD5B7DDE000000, 0x1BD5B7D, 0])); + let shr = shl >> 40; + assert_eq!(shr, U256([0x7DDE000000000000u64, 0x0001BD5B7DDFBD5B, 0, 0])); + // Increment + let incr = shr + U256::from(1u64); + assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); + // Subtraction + let sub = incr - init; + assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); + // Multiplication + let mult = sub.mul_u32(300); + assert_eq!(mult, U256([0x8C8C3EE70C644118u64, 0x0209E7378231E632, 0, 0])); + // Division + assert_eq!(U256::from(105u8) / U256::from(5u8), U256::from(21u8)); + let div = mult / U256::from(300u16); + assert_eq!(div, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); + //// TODO: bit inversion + } - #[test] - pub fn uint256_extreme_bitshift_test() { - //// Shifting a u64 by 64 bits gives an undefined value, so make sure that - //// we're doing the Right Thing here - let init = U256::from(0xDEADBEEFDEADBEEFu64); + #[test] + pub fn uint256_extreme_bitshift_test() { + //// Shifting a u64 by 64 bits gives an undefined value, so make sure that + //// we're doing the Right Thing here + let init = U256::from(0xDEADBEEFDEADBEEFu64); - assert_eq!(init << 64, U256([0, 0xDEADBEEFDEADBEEF, 0, 0])); - let add = (init << 64) + init; - assert_eq!(add, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add >> 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add << 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); - assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); - assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); - } + assert_eq!(init << 64, U256([0, 0xDEADBEEFDEADBEEF, 0, 0])); + let add = (init << 64) + init; + assert_eq!(add, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add >> 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add << 0, U256([0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0, 0])); + assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); + assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); + } } From 98feaa4f16cf222d6037d856867f4c25ee2a441a Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 03:39:13 +0100 Subject: [PATCH 220/379] docs --- src/lib.rs | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c52a1b2d0..1de5c7ca8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,31 @@ //! Ethcore-util library +//! +//! ### Rust version: +//! - beta +//! - nightly //! -//! TODO: check reexports +//! ### Supported platforms: +//! - OSX +//! - Linux +//! +//! ### Dependencies: +//! - RocksDB 3.13 +//! +//! ### Dependencies Installation: +//! +//! - OSX: +//! +//! ```bash +//! brew install rocksdb +//! ``` +//! +//! - From source: +//! +//! ```bash +//! wget https://github.com/facebook/rocksdb/archive/rocksdb-3.13.tar.gz +//! tar xvf rocksdb-3.13.tar.gz && cd rocksdb-rocksdb-3.13 && make shared_lib +//! sudo make install +//! ``` extern crate rustc_serialize; extern crate mio; @@ -39,12 +64,3 @@ pub mod trie; pub mod nibbleslice; //pub mod network; - -// reexports -pub use std::str::FromStr; -pub use hash::*; -pub use sha3::*; -pub use bytes::*; -pub use hashdb::*; -pub use memorydb::*; - From 2805dfe22dda728f062ca3f9535704c5ba7ca961 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 11:40:54 +0100 Subject: [PATCH 221/379] removed num library from dependencies --- Cargo.toml | 1 - src/chainfilter.rs | 17 ++++++++++++----- src/lib.rs | 1 - 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 41ca44580..65ea8508f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" rocksdb = "0.2.1" -num = "0.1" lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 538d1883f..076263fbb 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -41,7 +41,6 @@ use std::collections::{HashMap}; use hash::*; use sha3::*; -use num::pow; /// Represents bloom index in cache /// @@ -119,13 +118,21 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource let mut filter = ChainFilter { data_source: data_source, index_size: index_size, - level_sizes: vec![] + // 0 level has always a size of 1 + level_sizes: vec![1] }; // cache level sizes, so we do not have to calculate them all the time - for i in 0..levels { - filter.level_sizes.push(pow(index_size, i as usize)); - } + // eg. if levels == 3, index_size = 16 + // level_sizes = [1, 16, 256] + let additional: Vec = (1..).into_iter() + .scan(1, |acc, _| { + *acc = *acc * index_size; + Some(*acc) + }) + .take(levels as usize - 1) + .collect(); + filter.level_sizes.extend(additional); filter } diff --git a/src/lib.rs b/src/lib.rs index 1de5c7ca8..6366d682e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,7 +32,6 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; -extern crate num; #[macro_use] extern crate log; #[macro_use] From 94edc594d7d023f82eab9618619be4aaaf646132 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 4 Dec 2015 13:12:11 +0100 Subject: [PATCH 222/379] docs --- src/sha3.rs | 14 ++++++++++++++ src/triehash.rs | 3 +-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/sha3.rs b/src/sha3.rs index 11d7ca274..b3676720c 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,8 +1,22 @@ +//! Wrapper around tiny-keccak crate. + use std::mem::uninitialized; use tiny_keccak::Keccak; use bytes::BytesConvertable; use hash::{FixedHash, H256}; +/// Types implementing this trait are sha3able. +/// +/// ``` +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::sha3::*; +/// use util::hash::*; +/// +/// fn main() { +/// assert_eq!([0u8; 0].sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); +/// } +/// ``` pub trait Hashable { fn sha3(&self) -> H256; } diff --git a/src/triehash.rs b/src/triehash.rs index 93d21aacc..eea79e9d4 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -1,4 +1,4 @@ -//! Generete trie root. +//! Generetes trie root. //! //! This module should be used to generate trie root hash. @@ -10,7 +10,6 @@ use rlp; use rlp::RlpStream; use vector::SharedPrefix; -// todo: verify if example for ordered_trie_root is valid /// Generates a trie root hash for a vector of values /// /// ```rust From 53547adea8b72d36ad997894e32abf40b89bc809 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Dec 2015 18:05:59 +0100 Subject: [PATCH 223/379] Fix for trie. Benchmarks for Trie. --- benches/trie.rs | 128 ++++++++++++++++++++++++++++++++++++++ src/hashdb.rs | 2 +- src/memorydb.rs | 8 +-- src/overlaydb.rs | 12 ++-- src/trie.rs | 159 +++++++++++++++++++++++++++++++++++++---------- 5 files changed, 262 insertions(+), 47 deletions(-) create mode 100644 benches/trie.rs diff --git a/benches/trie.rs b/benches/trie.rs new file mode 100644 index 000000000..0b7055bde --- /dev/null +++ b/benches/trie.rs @@ -0,0 +1,128 @@ +#![feature(test)] + +extern crate test; +extern crate rand; +extern crate ethcore_util; +#[macro_use] +extern crate log; + +use test::Bencher; +use rand::random; +//use ethcore_util::BytesConvertable; +use ethcore_util::hash::*; +use ethcore_util::bytes::*; +use ethcore_util::trie::*; +use ethcore_util::sha3::*; +use ethcore_util::ToBytes::*; + + +fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let mut ret: Vec = Vec::with_capacity(r); + for i in 0..r { + ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); + } + ret +} + +fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + seed.bytes()[0..r].to_vec() +} + +fn random_value(seed: &mut H256) -> Bytes { + *seed = seed.sha3(); + match seed.bytes()[0] % 2 { + 1 => vec![seed.bytes()[31];1], + _ => seed.bytes().to_vec(), + } +} + +#[bench] +fn insertions_six_high(b: &mut Bencher) { + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_bytes(6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + }) +} + +#[bench] +fn insertions_six_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + debug!("hash_count={:?}", t.hash_count); + }) +} + +#[bench] +fn insertions_random_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 1, 5, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + }) +} + +#[bench] +fn insertions_six_low(b: &mut Bencher) { + let alphabet = b"abcdef"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + let mut t = TrieDB::new_memory(); + for i in d.iter() { + t.insert(&i.0, &i.1); + } + }) +} + +#[bench] +fn sha3x1000(b: &mut Bencher) { + b.iter(||{ + let mut seed = H256::new(); + for i in 0..1000 { + seed = seed.sha3() + } + }) +} diff --git a/src/hashdb.rs b/src/hashdb.rs index f893c8df8..207883e4b 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; pub trait HashDB { /// Get the keys in the database together with number of underlying references. - fn keys(&self) -> HashMap; + fn keys(&self) -> HashMap; /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. diff --git a/src/memorydb.rs b/src/memorydb.rs index 8c7eaff2c..680129670 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -116,10 +116,6 @@ impl MemoryDB { } self.data.get(key).unwrap() } - - pub fn raw_keys(&self) -> HashMap { - self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect::>() - } } impl HashDB for MemoryDB { @@ -130,8 +126,8 @@ impl HashDB for MemoryDB { } } - fn keys(&self) -> HashMap { - self.data.iter().filter_map(|(k, v)| if v.1 > 0 {Some((k.clone(), v.1 as u32))} else {None} ).collect::>() + fn keys(&self) -> HashMap { + self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect::>() } fn exists(&self, key: &H256) -> bool { diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 78ca67d01..c13acfd6a 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -136,17 +136,17 @@ impl OverlayDB { } impl HashDB for OverlayDB { - fn keys(&self) -> HashMap { - let mut ret: HashMap = HashMap::new(); + fn keys(&self) -> HashMap { + let mut ret: HashMap = HashMap::new(); for (key, _) in self.backing.iterator().from_start() { let h = H256::from_slice(key.deref()); let r = self.payload(&h).unwrap().1; - ret.insert(h, r); + ret.insert(h, r as i32); } - for (key, refs) in self.overlay.raw_keys().into_iter() { - let refs = *ret.get(&key).unwrap_or(&0u32) as i32 + refs as i32; - ret.insert(key, refs as u32); + for (key, refs) in self.overlay.keys().into_iter() { + let refs = *ret.get(&key).unwrap_or(&0) + refs; + ret.insert(key, refs); } ret } diff --git a/src/trie.rs b/src/trie.rs index 81ac4f99b..a340d78f7 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -26,6 +26,67 @@ pub trait Trie { fn remove(&mut self, key: &[u8]); } +pub enum Alphabet { + All, + Low, + Mid, + Custom(Bytes), +} + +pub struct StandardMap { + alphabet: Alphabet, + min_key: usize, + diff_key: usize, + count: usize, +} + +impl StandardMap { + fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + seed.bytes()[0..r].to_vec() + } + + fn random_value(seed: &mut H256) -> Bytes { + *seed = seed.sha3(); + match seed.bytes()[0] % 2 { + 1 => vec![seed.bytes()[31];1], + _ => seed.bytes().to_vec(), + } + } + + fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + diff_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let mut ret: Vec = Vec::with_capacity(r); + for i in 0..r { + ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); + } + ret + } + + pub fn make(&self) -> Vec<(Bytes, Bytes)> { + let low = b"abcdef"; + let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..self.count { + let k = match self.alphabet { + Alphabet::All => Self::random_bytes(self.min_key, self.diff_key, &mut seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.diff_key, &mut seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.diff_key, &mut seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.diff_key, &mut seed), + }; + let v = Self::random_value(&mut seed); + d.push((k, v)) + } + d + } +} + #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { Empty, @@ -34,11 +95,13 @@ pub enum Node<'a> { Branch([&'a[u8]; 16], Option<&'a [u8]>) } +#[derive(Debug)] enum Operation { New(H256, Bytes), Delete(H256), } +#[derive(Debug)] struct Diff (Vec); impl Diff { @@ -48,8 +111,9 @@ impl Diff { /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { if rlp.len() >= 32 { - trace!("new_node: reference node {:?}", rlp.pretty()); let rlp_sha3 = rlp.sha3(); + + trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty()); out.append(&rlp_sha3); self.0.push(Operation::New(rlp_sha3, rlp)); } @@ -61,25 +125,18 @@ impl Diff { /// Given the RLP that encodes a now-unused node, leave `diff` in such a state that it is noted. fn delete_node_sha3(&mut self, old_sha3: H256) { + trace!("delete_node: {:?}", old_sha3); self.0.push(Operation::Delete(old_sha3)); } fn delete_node(&mut self, old: &Rlp) { if old.is_data() && old.size() == 32 { - self.0.push(Operation::Delete(H256::decode(old))); + self.delete_node_sha3(H256::decode(old)); } } fn delete_node_from_slice(&mut self, old: &[u8]) { - let r = Rlp::new(old); - if r.is_data() && r.size() == 32 { - self.0.push(Operation::Delete(H256::decode(&r))); - } - } - - fn replace_node(&mut self, old: &Rlp, rlp: Bytes, out: &mut RlpStream) { - self.delete_node(old); - self.new_node(rlp, out); + self.delete_node(&Rlp::new(old)); } } @@ -192,11 +249,12 @@ impl <'a>Node<'a> { pub struct TrieDB { db: Box, root: H256, + pub hash_count: usize, } impl fmt::Debug for TrieDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - try!(writeln!(f, "[")); + try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); writeln!(f, "]") @@ -209,7 +267,7 @@ enum MaybeChanged<'a> { } impl TrieDB { - pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new() }; r.set_root_rlp(&NULL_RLP); r } + pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new(), hash_count: 0 }; r.root = r.db.insert(&NULL_RLP); r } pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } @@ -220,6 +278,7 @@ impl TrieDB { fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); + self.hash_count += 1; trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } @@ -234,6 +293,7 @@ impl TrieDB { Operation::New(h, d) => { trace!("TrieDB::apply +++ {:?} -> {:?}", &h, d.pretty()); self.db.emplace(h, d); + self.hash_count += 1; } } } @@ -272,14 +332,13 @@ impl TrieDB { r } - pub fn db_items_remaining(&self) -> HashMap { + pub fn db_items_remaining(&self) -> HashMap { let mut ret = self.db().keys(); for (k, v) in Self::to_map(self.keys()).into_iter() { - let old = *ret.get(&k).expect("Node in trie is not in database!"); - assert!(old >= v); - match old > v { - true => ret.insert(k, old - v), - _ => ret.remove(&k), + let keycount = *ret.get(&k).unwrap_or(&0); + match keycount == v as i32 { + true => ret.remove(&k), + _ => ret.insert(k, keycount - v as i32), }; } ret @@ -437,7 +496,11 @@ impl TrieDB { } else if rlp.is_data() && rlp.size() == 32 { let h = H256::decode(rlp); - let r = self.db.lookup(&h).expect("Trie root not found!"); + let r = self.db.lookup(&h).unwrap_or_else(||{ + println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); + println!("Diff: {:?}", diff); + panic!(); + }); trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); diff.delete_node_sha3(h); r @@ -518,7 +581,7 @@ impl TrieDB { diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), (true, i) => { // harder - original has something there already let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); - diff.replace_node(&orig.at(i), new, &mut s); + diff.new_node(new, &mut s); } (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, } @@ -804,22 +867,21 @@ mod tests { use std::collections::HashSet; use bytes::{ToPretty,Bytes}; - fn random_key() -> Vec { - let chars = b"abcdefgrstuvwABCDEFGRSTUVW"; + fn random_key(alphabet: &[u8], min_count: usize, diff_count: usize) -> Vec { let mut ret: Vec = Vec::new(); - let r = random::() % 4 + 1; + let r = min_count + if diff_count > 0 {random::() % diff_count} else {0}; for _ in 0..r { - ret.push(chars[random::() % chars.len()]); + ret.push(alphabet[random::() % alphabet.len()]); } ret } - - fn random_value(i: usize) -> Bytes { + + fn random_value_indexed(j: usize) -> Bytes { match random::() % 2 { - 0 => rlp::encode(&i), + 0 => rlp::encode(&j), _ => { let mut h = H256::new(); - h.mut_bytes()[31] = i as u8; + h.mut_bytes()[31] = j as u8; rlp::encode(&h) }, } @@ -842,16 +904,44 @@ mod tests { } } + macro_rules! map({$($key:expr => $value:expr),+ } => { + { + let mut m = ::std::collections::HashMap::new(); + $( + m.insert($key, $value); + )+ + m + } + };); + #[test] fn playpen() { env_logger::init().ok(); - for _ in 0..1000 { + + let maps = map!{ + "six-low" => StandardMap{alphabet: Alphabet::Low, min_key: 6, diff_key: 0, count: 1000}, + "six-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 6, diff_key: 0, count: 1000}, + "six-all" => StandardMap{alphabet: Alphabet::All, min_key: 6, diff_key: 0, count: 1000}, + "mix-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 1, diff_key: 5, count: 1000} + }; + for sm in maps { + let m = sm.1.make(); + let t = populate_trie(&m); + println!("{:?}: root={:?}, hash_count={:?}", sm.0, t.root(), t.hash_count); + }; + panic!(); + + for test_i in 0..1 { + if test_i % 50 == 0 { + debug!("{:?} of 10000 stress tests done", test_i); + } let mut x: Vec<(Vec, Vec)> = Vec::new(); let mut got: HashSet> = HashSet::new(); - for j in 0..10usize { - let key = random_key(); + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + for j in 0..1000usize { + let key = random_key(alphabet, 5, 0); if !got.contains(&key) { - x.push((key.clone(), random_value(j))); + x.push((key.clone(), random_value_indexed(j))); got.insert(key); } } @@ -1101,8 +1191,9 @@ mod tests { fn stress() { for _ in 0..5000 { let mut x: Vec<(Vec, Vec)> = Vec::new(); + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; for j in 0..4u32 { - let key = random_key(); + let key = random_key(alphabet, 5, 1); x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); From 38f0af1aa0e47195d74a9552cc0530945effe6ca Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 4 Dec 2015 20:04:26 +0100 Subject: [PATCH 224/379] Code cleanups. --- src/trie.rs | 87 +++++++++++++++++------------------------------------ 1 file changed, 27 insertions(+), 60 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index a340d78f7..48fbe8d83 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -511,58 +511,24 @@ impl TrieDB { } } - /// Transform an existing extension or leaf node to an invalid single-entry branch. - /// - /// **This operation will not insert the new node nor destroy the original.** - fn transmuted_extension_to_branch(orig_partial: &NibbleSlice, orig_raw_payload: &[u8], diff: &mut Diff) -> Bytes { - trace!("transmuted_extension_to_branch"); - let mut s = RlpStream::new_list(17); - assert!(!orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. - let index = orig_partial.at(0); - // orig is extension - orig_raw_payload is a node itself. - for i in 0..17 { - if index == i { - if orig_partial.len() > 1 { - // still need an extension - diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s); - } else { - // was an extension of length 1 - just redirect the payload into here. - s.append_raw(orig_raw_payload, 1); - } - } else { - s.append_empty_data(); - } - } - s.out() - } - - fn transmuted_leaf_to_branch(orig_partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("transmuted_leaf_to_branch"); + fn augmented_into_transmuted_branch(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + assert!(orig_is_leaf || !orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. let mut s = RlpStream::new_list(17); let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; - // orig is leaf - orig_raw_payload is data representing the actual value. for i in 0..17 { - match (index == i, i) { - (true, 16) => // leaf entry - just replace. - { s.append(&value); }, - (true, _) => // easy - original had empty slot. - diff.new_node(Self::compose_leaf(&orig_partial.mid(1), value), &mut s), - (false, _) => { s.append_empty_data(); } + match orig_is_leaf { + // not us - empty. + _ if index != i => { s.append_empty_data(); }, + // just replace. + true if i == 16 => { s.append(&value); }, + // original has empty slot. + true => diff.new_node(Self::compose_leaf(&orig_partial.mid(1), Rlp::new(orig_raw_payload).data()), &mut s), + // + false if orig_partial.len() > 1 => diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s), + false => { s.append_raw(orig_raw_payload, 1); }, } - } - s.out() - } - - /// Transform an existing extension or leaf node plus a new partial/value to a two-entry branch. - /// - /// **This operation will not insert the new node nor destroy the original.** - fn transmuted_to_branch_and_augmented(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("transmuted_to_branch_and_augmented"); - let intermediate = match orig_is_leaf { - true => Self::transmuted_leaf_to_branch(orig_partial, Rlp::new(orig_raw_payload).data(), diff), - false => Self::transmuted_extension_to_branch(orig_partial, orig_raw_payload, diff), }; - self.augmented(&intermediate, partial, value, diff) + self.augmented(&s.out(), partial, value, diff) // TODO: implement without having to make an intermediate representation. } @@ -573,17 +539,19 @@ impl TrieDB { trace!("augmented_into_branch"); let mut s = RlpStream::new_list(17); let index = if partial.is_empty() {16} else {partial.at(0) as usize}; - for i in 0usize..17 { - match (index == i, i) { - (true, 16) => // leaf entry - just replace. - { s.append(&value); }, - (true, i) if orig.at(i).is_empty() => // easy - original had empty slot. - diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), - (true, i) => { // harder - original has something there already + for i in 0..17 { + match index == i { + // not us - leave alone. + false => { s.append_raw(orig.at(i).raw(), 1); }, + // branch-leaf entry - just replace. + true if i == 16 => { s.append(&value); }, + // original had empty slot - place a leaf there. + true if orig.at(i).is_empty() => diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), + // original has something there already; augment. + true => { let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); diff.new_node(new, &mut s); } - (false, i) => { s.append_raw(orig.at(i).raw(), 1); }, } } s.out() @@ -619,17 +587,16 @@ impl TrieDB { (_, 0) => { // one of us isn't empty: transmute to branch here trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); - self.transmuted_to_branch_and_augmented(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) + self.augmented_into_transmuted_branch(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) }, (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); // fully-shared prefix for this extension: // transform to an extension + augmented version of onward node. - let downstream_node: Bytes = if is_leaf { + let downstream_node: Bytes = match is_leaf { // no onward node because we're a leaf - create fake stub and use that. - self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff) - } else { - self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff) + true => self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff), + false => self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff), }; Self::create_extension(&existing_key, downstream_node, diff) }, From 5ba8f528ba3e10c8a45a9c55c48d0d778999307b Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 5 Dec 2015 13:22:22 +0100 Subject: [PATCH 225/379] removed build warnings and errors --- benches/trie.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index 0b7055bde..2865c5778 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -7,13 +7,10 @@ extern crate ethcore_util; extern crate log; use test::Bencher; -use rand::random; -//use ethcore_util::BytesConvertable; use ethcore_util::hash::*; use ethcore_util::bytes::*; use ethcore_util::trie::*; use ethcore_util::sha3::*; -use ethcore_util::ToBytes::*; fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { @@ -121,7 +118,7 @@ fn insertions_six_low(b: &mut Bencher) { fn sha3x1000(b: &mut Bencher) { b.iter(||{ let mut seed = H256::new(); - for i in 0..1000 { + for _ in 0..1000 { seed = seed.sha3() } }) From 9b080c91619decc61a0d7b0fb2bafaf47d6383ae Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 6 Dec 2015 11:20:37 +0100 Subject: [PATCH 226/379] microoptimization --- src/rlp.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index a7543aae5..bada78de0 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -844,8 +844,12 @@ impl RlpStream { // we may finish, if the appended list len is equal 0 self.encoder.bytes.push(0xc0u8); self.note_appended(1); - } - _ => self.unfinished_lists.push_back(ListInfo::new(position, len)), + }, + _ => { + // reserve at least double size of the len + self.encoder.bytes.reserve(len * 2); + self.unfinished_lists.push_back(ListInfo::new(position, len)); + }, } // return chainable self From 182f356467213fbf80e4a224fa4b4e027eb9a3de Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 6 Dec 2015 11:42:15 +0100 Subject: [PATCH 227/379] replace linked list with vec --- src/rlp.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/rlp.rs b/src/rlp.rs index bada78de0..624254fc6 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -32,7 +32,6 @@ use std::fmt; use std::cell::Cell; -use std::collections::LinkedList; use std::error::Error as StdError; use bytes::{ToBytes, FromBytes, FromBytesError}; use vector::InsertSlice; @@ -778,7 +777,7 @@ impl ListInfo { /// Appendable rlp encoder. pub struct RlpStream { - unfinished_lists: LinkedList, + unfinished_lists: Vec, encoder: BasicEncoder, } @@ -786,7 +785,7 @@ impl RlpStream { /// Initializes instance of empty `RlpStream`. pub fn new() -> RlpStream { RlpStream { - unfinished_lists: LinkedList::new(), + unfinished_lists: vec![], encoder: BasicEncoder::new(), } } @@ -848,7 +847,7 @@ impl RlpStream { _ => { // reserve at least double size of the len self.encoder.bytes.reserve(len * 2); - self.unfinished_lists.push_back(ListInfo::new(position, len)); + self.unfinished_lists.push(ListInfo::new(position, len)); }, } @@ -930,7 +929,7 @@ impl RlpStream { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } pub fn is_finished(&self) -> bool { - self.unfinished_lists.back().is_none() + self.unfinished_lists.len() == 0 } /// Streams out encoded bytes. @@ -945,7 +944,12 @@ impl RlpStream { /// Try to finish lists fn note_appended(&mut self, inserted_items: usize) -> () { - let should_finish = match self.unfinished_lists.back_mut() { + if self.unfinished_lists.len() == 0 { + return; + } + + let back = self.unfinished_lists.len() - 1; + let should_finish = match self.unfinished_lists.get_mut(back) { None => false, Some(ref mut x) => { x.current += inserted_items; @@ -957,7 +961,7 @@ impl RlpStream { }; if should_finish { - let x = self.unfinished_lists.pop_back().unwrap(); + let x = self.unfinished_lists.pop().unwrap(); let len = self.encoder.bytes.len() - x.position; self.encoder.insert_list_len_at_pos(len, x.position); self.note_appended(1); From 8cf437b34942a99798d13357859079032c371e89 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 7 Dec 2015 11:27:12 +0100 Subject: [PATCH 228/379] rlp uses elastic array --- Cargo.toml | 1 + src/lib.rs | 1 + src/rlp.rs | 29 +++++++++++++++-------------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 65ea8508f..13295f766 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ rocksdb = "0.2.1" lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" +elastic-array = "0.4" [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/lib.rs b/src/lib.rs index 6366d682e..883cac2f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,7 @@ extern crate time; extern crate crypto as rcrypto; extern crate secp256k1; extern crate arrayvec; +extern crate elastic_array; pub mod macros; pub mod error; diff --git a/src/rlp.rs b/src/rlp.rs index 624254fc6..d0dece3ba 100644 --- a/src/rlp.rs +++ b/src/rlp.rs @@ -33,6 +33,7 @@ use std::fmt; use std::cell::Cell; use std::error::Error as StdError; +use elastic_array::*; use bytes::{ToBytes, FromBytes, FromBytesError}; use vector::InsertSlice; @@ -758,7 +759,7 @@ impl Decoder for BasicDecoder { } } -#[derive(Debug)] +#[derive(Debug, Copy, Clone)] struct ListInfo { position: usize, current: usize, @@ -777,7 +778,7 @@ impl ListInfo { /// Appendable rlp encoder. pub struct RlpStream { - unfinished_lists: Vec, + unfinished_lists: ElasticArray16, encoder: BasicEncoder, } @@ -785,7 +786,7 @@ impl RlpStream { /// Initializes instance of empty `RlpStream`. pub fn new() -> RlpStream { RlpStream { - unfinished_lists: vec![], + unfinished_lists: ElasticArray16::new(), encoder: BasicEncoder::new(), } } @@ -846,7 +847,7 @@ impl RlpStream { }, _ => { // reserve at least double size of the len - self.encoder.bytes.reserve(len * 2); + //self.encoder.bytes.reserve(len * 2); self.unfinished_lists.push(ListInfo::new(position, len)); }, } @@ -882,7 +883,7 @@ impl RlpStream { /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { // push raw items - self.encoder.bytes.extend(bytes); + self.encoder.bytes.append_slice(bytes); // try to finish and prepend the length self.note_appended(item_count); @@ -937,7 +938,7 @@ impl RlpStream { /// panic! if stream is not finished. pub fn out(self) -> Vec { match self.is_finished() { - true => self.encoder.out(), + true => self.encoder.out().to_vec(), false => panic!() } } @@ -985,7 +986,7 @@ pub fn encode(object: &E) -> Vec where E: Encodable { let mut encoder = BasicEncoder::new(); object.encode(&mut encoder); - encoder.out() + encoder.out().to_vec() } pub trait Encodable { @@ -1038,12 +1039,12 @@ impl Encodable for Vec { } struct BasicEncoder { - bytes: Vec, + bytes: ElasticArray1024, } impl BasicEncoder { fn new() -> BasicEncoder { - BasicEncoder { bytes: vec![] } + BasicEncoder { bytes: ElasticArray1024::new() } } /// inserts list prefix at given position @@ -1062,7 +1063,7 @@ impl BasicEncoder { } /// get encoded value - fn out(self) -> Vec { + fn out(self) -> ElasticArray1024 { self.bytes } } @@ -1073,17 +1074,17 @@ impl Encoder for BasicEncoder { // just 0 0 => self.bytes.push(0x80u8), // byte is its own encoding - 1 if bytes[0] < 0x80 => self.bytes.extend(bytes), + 1 if bytes[0] < 0x80 => self.bytes.append_slice(bytes), // (prefix + length), followed by the string len @ 1 ... 55 => { self.bytes.push(0x80u8 + len as u8); - self.bytes.extend(bytes); + self.bytes.append_slice(bytes); } // (prefix + length of length), followed by the length, followd by the string len => { self.bytes.push(0xb7 + len.to_bytes_len() as u8); - self.bytes.extend(len.to_bytes()); - self.bytes.extend(bytes); + self.bytes.append_slice(&len.to_bytes()); + self.bytes.append_slice(bytes); } } } From fcae9034acbd17e136f5a062c3d4bc095c56bb3f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 7 Dec 2015 15:00:54 +0100 Subject: [PATCH 229/379] Trie documentation and code consolidation. --- src/trie.rs | 451 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 265 insertions(+), 186 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 48fbe8d83..0425565bc 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -15,17 +15,30 @@ use std::collections::HashMap; pub const NULL_RLP: [u8; 1] = [0x80; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); +/// A key-value datastore implemented as a database-backed modified Merkle tree. pub trait Trie { + /// Return the root of the trie. fn root(&self) -> &H256; + + /// Is the trie empty? fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } - // TODO: consider returning &[u8]... + /// Does the trie contain a given key? fn contains(&self, key: &[u8]) -> bool; + + /// What is the value of the given key in this trie? fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; + + /// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing + /// `key` from the trie. fn insert(&mut self, key: &[u8], value: &[u8]); + + /// Remove a `key` from the trie. Equivalent to making it equal to the empty + /// value. fn remove(&mut self, key: &[u8]); } +/// Alphabet to use when creating words for insertion into tries. pub enum Alphabet { All, Low, @@ -33,21 +46,25 @@ pub enum Alphabet { Custom(Bytes), } +/// Standard test map for profiling tries. pub struct StandardMap { alphabet: Alphabet, min_key: usize, - diff_key: usize, + journal_key: usize, count: usize, } impl StandardMap { - fn random_bytes(min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { - assert!(min_count + diff_count <= 32); + /// Get a bunch of random bytes, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. + /// `seed` is mutated pseudoramdonly and used. + fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + journal_count <= 32); *seed = seed.sha3(); - let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); seed.bytes()[0..r].to_vec() } + /// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used. fn random_value(seed: &mut H256) -> Bytes { *seed = seed.sha3(); match seed.bytes()[0] % 2 { @@ -56,10 +73,12 @@ impl StandardMap { } } - fn random_word(alphabet: &[u8], min_count: usize, diff_count: usize, seed: &mut H256) -> Vec { - assert!(min_count + diff_count <= 32); + /// Get a random word of, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. + /// Each byte is an item from `alphabet`. `seed` is mutated pseudoramdonly and used. + fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + journal_count <= 32); *seed = seed.sha3(); - let r = min_count + (seed.bytes()[31] as usize % (diff_count + 1)); + let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); let mut ret: Vec = Vec::with_capacity(r); for i in 0..r { ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); @@ -67,6 +86,7 @@ impl StandardMap { ret } + /// Create the standard map (set of keys and values) for the object's fields. pub fn make(&self) -> Vec<(Bytes, Bytes)> { let low = b"abcdef"; let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; @@ -75,10 +95,10 @@ impl StandardMap { let mut seed = H256::new(); for _ in 0..self.count { let k = match self.alphabet { - Alphabet::All => Self::random_bytes(self.min_key, self.diff_key, &mut seed), - Alphabet::Low => Self::random_word(low, self.min_key, self.diff_key, &mut seed), - Alphabet::Mid => Self::random_word(mid, self.min_key, self.diff_key, &mut seed), - Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.diff_key, &mut seed), + Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed), }; let v = Self::random_value(&mut seed); d.push((k, v)) @@ -87,6 +107,7 @@ impl StandardMap { } } +/// Type of node in the trie and essential information thereof. #[derive(Eq, PartialEq, Debug)] pub enum Node<'a> { Empty, @@ -95,19 +116,22 @@ pub enum Node<'a> { Branch([&'a[u8]; 16], Option<&'a [u8]>) } +/// Type of operation for the backing database - either a new node or a node deletion. #[derive(Debug)] enum Operation { New(H256, Bytes), Delete(H256), } +/// A journal of operations on the backing database. #[derive(Debug)] -struct Diff (Vec); +struct Journal (Vec); -impl Diff { - fn new() -> Diff { Diff(vec![]) } +impl Journal { + /// Create a new, empty, object. + fn new() -> Journal { Journal(vec![]) } - /// Given the RLP that encodes a node, append a reference to that node `out` and leave `diff` + /// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal` /// such that the reference is valid, once applied. fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { if rlp.len() >= 32 { @@ -123,24 +147,23 @@ impl Diff { } } - /// Given the RLP that encodes a now-unused node, leave `diff` in such a state that it is noted. + /// Given the RLP that encodes a now-unused node, leave `journal` in such a state that it is noted. fn delete_node_sha3(&mut self, old_sha3: H256) { trace!("delete_node: {:?}", old_sha3); self.0.push(Operation::Delete(old_sha3)); } - fn delete_node(&mut self, old: &Rlp) { - if old.is_data() && old.size() == 32 { - self.delete_node_sha3(H256::decode(old)); + /// Register an RLP-encoded node for deletion (given a slice), if it needs to be deleted. + fn delete_node(&mut self, old: &[u8]) { + let r = Rlp::new(old); + if r.is_data() && r.size() == 32 { + self.delete_node_sha3(H256::decode(&r)); } } - - fn delete_node_from_slice(&mut self, old: &[u8]) { - self.delete_node(&Rlp::new(old)); - } } impl <'a>Node<'a> { + /// Decode the `node_rlp` and return the Node. fn decoded(node_rlp: &'a [u8]) -> Node<'a> { let r = Rlp::new(node_rlp); match r.prototype() { @@ -168,7 +191,10 @@ impl <'a>Node<'a> { } } - // todo: should check length before encoding, cause it may just be sha3 of data + /// Encode the node into RLP. + /// + /// Will always return the direct node RLP even if it's 32 or more bytes. To get the + /// RLP which would be valid for using in another node, use `encoded_and_added()`. fn encoded(&self) -> Bytes { match *self { Node::Leaf(ref slice, ref value) => { @@ -202,7 +228,9 @@ impl <'a>Node<'a> { } } - fn encoded_and_added(&self, diff: &mut Diff) -> Bytes { + /// Encode the node, adding it to `journal` if necessary and return the RLP valid for + /// insertion into a parent node. + fn encoded_and_added(&self, journal: &mut Journal) -> Bytes { let mut stream = RlpStream::new(); match *self { Node::Leaf(ref slice, ref value) => { @@ -234,47 +262,96 @@ impl <'a>Node<'a> { 0 ... 31 => node, _ => { let mut stream = RlpStream::new(); - diff.new_node(node, &mut stream); + journal.new_node(node, &mut stream); stream.out() } } } } -//enum ValidationResult<'a> { - //Valid, - //Invalid { node: Node<'a>, depth: usize } -//} - +/// A `Trie` implementation using a generic `HashDB` backing database. +/// +/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` +/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get +/// which items in the backing database do not belong to this trie. If this is the only trie in the +/// backing database, then `db_items_remaining()` should be empty. +/// +/// # Example +/// ``` +/// extern crate ethcore_util; +/// use ethcore_util::trie::*; +/// fn main() { +/// let mut t = TrieDB::new_memory(); +/// assert!(t.is_empty()); +/// assert_eq!(*t.root(), SHA3_NULL_RLP); +/// t.insert(b"foo", b"bar"); +/// assert!(t.contains(b"foo")); +/// assert_eq!(t.at(b"foo").unwrap(), b"bar"); +/// assert!(t.db_items_remaining().is_empty()); +/// t.remove(b"foo"); +/// assert!(!t.contains(b"foo")); +/// assert!(t.db_items_remaining().is_empty()); +/// } +/// ``` pub struct TrieDB { db: Box, root: H256, pub hash_count: usize, } -impl fmt::Debug for TrieDB { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - try!(writeln!(f, "c={:?} [", self.hash_count)); - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); - writeln!(f, "]") - } -} - +/// Option-like type allowing either a Node object passthrough or Bytes in the case of data alteration. enum MaybeChanged<'a> { Same(Node<'a>), Changed(Bytes), } impl TrieDB { + /// Create a new trie with the boxed backing database `box_db`. pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new(), hash_count: 0 }; r.root = r.db.insert(&NULL_RLP); r } + /// Convenience function to create a new trie with the backing database `db`. pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } + /// Convenience function to create a new trie with a new `MemoryDB` based backing database. pub fn new_memory() -> Self { Self::new(MemoryDB::new()) } + /// Get the backing database. pub fn db(&self) -> &HashDB { self.db.as_ref() } + /// Determine all the keys in the backing database that belong to the trie. + pub fn keys(&self) -> Vec { + let mut ret: Vec = Vec::new(); + ret.push(self.root.clone()); + self.accumulate_keys(self.root_node(), &mut ret); + ret + } + + /// Convert a vector of hashes to a hashmap of hash to occurances. + pub fn to_map(hashes: Vec) -> HashMap { + let mut r: HashMap = HashMap::new(); + for h in hashes.into_iter() { + let c = *r.get(&h).unwrap_or(&0); + r.insert(h, c + 1); + } + r + } + + /// Determine occurances of items in the backing database which are not related to this + /// trie. + pub fn db_items_remaining(&self) -> HashMap { + let mut ret = self.db().keys(); + for (k, v) in Self::to_map(self.keys()).into_iter() { + let keycount = *ret.get(&k).unwrap_or(&0); + match keycount == v as i32 { + true => ret.remove(&k), + _ => ret.insert(k, keycount - v as i32), + }; + } + ret + } + + /// Set the trie to a new root node's RLP, inserting the new RLP into the backing database + /// and removing the old. fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); self.root = self.db.insert(root_data); @@ -282,9 +359,10 @@ impl TrieDB { trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } - fn apply(&mut self, diff: Diff) { - trace!("applying {:?} changes", diff.0.len()); - for d in diff.0.into_iter() { + /// Apply the items in `journal` into the backing database. + fn apply(&mut self, journal: Journal) { + trace!("applying {:?} changes", journal.0.len()); + for d in journal.0.into_iter() { match d { Operation::Delete(h) => { trace!("TrieDB::apply --- {:?}", &h); @@ -299,13 +377,7 @@ impl TrieDB { } } - pub fn keys(&self) -> Vec { - let mut ret: Vec = Vec::new(); - ret.push(self.root.clone()); - self.accumulate_keys(self.root_node(), &mut ret); - ret - } - + /// Recursion helper for `keys`. fn accumulate_keys(&self, node: Node, acc: &mut Vec) { let mut handle_payload = |payload| { let p = Rlp::new(payload); @@ -323,27 +395,17 @@ impl TrieDB { } } - fn to_map(hashes: Vec) -> HashMap { - let mut r: HashMap = HashMap::new(); - for h in hashes.into_iter() { - let c = *r.get(&h).unwrap_or(&0); - r.insert(h, c + 1); - } - r + /// Get the root node's RLP. + fn root_node(&self) -> Node { + Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) } - pub fn db_items_remaining(&self) -> HashMap { - let mut ret = self.db().keys(); - for (k, v) in Self::to_map(self.keys()).into_iter() { - let keycount = *ret.get(&k).unwrap_or(&0); - match keycount == v as i32 { - true => ret.remove(&k), - _ => ret.insert(k, keycount - v as i32), - }; - } - ret + /// Get the root node as a `Node`. + fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { + Node::decoded(self.get_raw_or_lookup(node)) } + /// Indentation helper for `formal_all`. fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { for _ in 0..size { try!(write!(f, " ")); @@ -351,14 +413,7 @@ impl TrieDB { Ok(()) } - fn root_node(&self) -> Node { - Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) - } - - fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { - Node::decoded(self.get_raw_or_lookup(node)) - } - + /// Recursion helper for implementation of formatting trait. fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { match node { Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())), @@ -394,11 +449,16 @@ impl TrieDB { Ok(()) } + /// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists. fn get<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); self.get_from_node(&root_rlp, key) } + /// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no + /// value exists for the key. + /// + /// Note: Not a public API; use Trie trait functions. fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { match Node::decoded(node) { Node::Leaf(ref slice, ref value) if key == slice => Some(value), @@ -413,6 +473,9 @@ impl TrieDB { } } + /// Given some node-describing data `node`, return the actual node RLP. + /// This could be a simple identity operation in the case that the node is sufficiently small, but + /// may require a database lookup. fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { // check if its sha3 + len let r = Rlp::new(node); @@ -422,20 +485,26 @@ impl TrieDB { } } - fn add(&mut self, key: &NibbleSlice, value: &[u8]) { + /// Insert a `key` and `value` pair into the trie. + /// + /// Note: Not a public API; use Trie trait functions. + fn insert_ns(&mut self, key: &NibbleSlice, value: &[u8]) { trace!("ADD: {:?} {:?}", key, value.pretty()); // determine what the new root is, insert new nodes and remove old as necessary. - let mut todo: Diff = Diff::new(); + let mut todo: Journal = Journal::new(); let root_rlp = self.augmented(self.db.lookup(&self.root).expect("Trie root not found!"), key, value, &mut todo); self.apply(todo); self.set_root_rlp(&root_rlp); trace!("/"); } - fn delete(&mut self, key: &NibbleSlice) { + /// Remove a `key` and `value` pair from the trie. + /// + /// Note: Not a public API; use Trie trait functions. + fn remove_ns(&mut self, key: &NibbleSlice) { trace!("DELETE: {:?}", key); // determine what the new root is, insert new nodes and remove old as necessary. - let mut todo: Diff = Diff::new(); + let mut todo: Journal = Journal::new(); match self.cleared_from_slice(self.db.lookup(&self.root).expect("Trie root not found!"), key, &mut todo) { Some(root_rlp) => { self.apply(todo); @@ -448,6 +517,7 @@ impl TrieDB { trace!("/"); } + /// Compose a leaf node in RLP given the `partial` key and `value`. fn compose_leaf(partial: &NibbleSlice, value: &[u8]) -> Bytes { trace!("compose_leaf {:?} {:?} ({:?})", partial, value.pretty(), partial.encoded(true).pretty()); let mut s = RlpStream::new_list(2); @@ -458,6 +528,7 @@ impl TrieDB { r } + /// Compose a raw extension/leaf node in RLP given the `partial` key, `raw_payload` and whether it `is_leaf`. fn compose_raw(partial: &NibbleSlice, raw_payload: &[u8], is_leaf: bool) -> Bytes { trace!("compose_raw {:?} {:?} {:?} ({:?})", partial, raw_payload.pretty(), is_leaf, partial.encoded(is_leaf)); let mut s = RlpStream::new_list(2); @@ -468,6 +539,7 @@ impl TrieDB { r } + /// Compose a branch node in RLP with a particular `value` sitting in the value position (17th place). fn compose_stub_branch(value: &[u8]) -> Bytes { let mut s = RlpStream::new_list(17); for _ in 0..16 { s.append_empty_data(); } @@ -475,21 +547,14 @@ impl TrieDB { s.out() } + /// Compose an extension node's RLP with the `partial` key and `raw_payload`. fn compose_extension(partial: &NibbleSlice, raw_payload: &[u8]) -> Bytes { Self::compose_raw(partial, raw_payload, false) } - fn create_extension(partial: &NibbleSlice, downstream_node: Bytes, diff: &mut Diff) -> Bytes { - trace!("create_extension partial: {:?}, downstream_node: {:?}", partial, downstream_node.pretty()); - let mut s = RlpStream::new_list(2); - s.append(&partial.encoded(false)); - diff.new_node(downstream_node, &mut s); - s.out() - } - - /// Return the bytes encoding the node represented by `rlp`. It will be unlinked from - /// the trie. - fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, diff: &mut Diff) -> &'a [u8] where 'a: 'rlp_view { + /// Return the bytes encoding the node represented by `rlp`. `journal` will record necessary + /// removal instructions from the backing database. + fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, journal: &mut Journal) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { trace!("take_node {:?} (inline)", rlp.raw().pretty()); rlp.raw() @@ -498,11 +563,11 @@ impl TrieDB { let h = H256::decode(rlp); let r = self.db.lookup(&h).unwrap_or_else(||{ println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); - println!("Diff: {:?}", diff); + println!("Journal: {:?}", journal); panic!(); }); trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); - diff.delete_node_sha3(h); + journal.delete_node_sha3(h); r } else { @@ -511,61 +576,15 @@ impl TrieDB { } } - fn augmented_into_transmuted_branch(&self, orig_is_leaf: bool, orig_partial: &NibbleSlice, orig_raw_payload: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - assert!(orig_is_leaf || !orig_partial.is_empty()); // extension nodes are not allowed to have empty partial keys. - let mut s = RlpStream::new_list(17); - let index = if orig_partial.is_empty() {16} else {orig_partial.at(0)}; - for i in 0..17 { - match orig_is_leaf { - // not us - empty. - _ if index != i => { s.append_empty_data(); }, - // just replace. - true if i == 16 => { s.append(&value); }, - // original has empty slot. - true => diff.new_node(Self::compose_leaf(&orig_partial.mid(1), Rlp::new(orig_raw_payload).data()), &mut s), - // - false if orig_partial.len() > 1 => diff.new_node(Self::compose_extension(&orig_partial.mid(1), orig_raw_payload), &mut s), - false => { s.append_raw(orig_raw_payload, 1); }, - } - }; - self.augmented(&s.out(), partial, value, diff) - // TODO: implement without having to make an intermediate representation. - } - - /// Given a branch node's RLP `orig` together with a `partial` key and `value`, return the - /// RLP-encoded node that accomodates the trie with the new entry. Mutate `diff` so that - /// once applied the returned node is valid. - fn augmented_into_branch(&self, orig: &Rlp, partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { - trace!("augmented_into_branch"); - let mut s = RlpStream::new_list(17); - let index = if partial.is_empty() {16} else {partial.at(0) as usize}; - for i in 0..17 { - match index == i { - // not us - leave alone. - false => { s.append_raw(orig.at(i).raw(), 1); }, - // branch-leaf entry - just replace. - true if i == 16 => { s.append(&value); }, - // original had empty slot - place a leaf there. - true if orig.at(i).is_empty() => diff.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), - // original has something there already; augment. - true => { - let new = self.augmented(self.take_node(&orig.at(i), diff), &partial.mid(1), value, diff); - diff.new_node(new, &mut s); - } - } - } - s.out() - } - /// Determine the RLP of the node, assuming we're inserting `partial` into the /// node currently of data `old`. This will *not* delete any hash of `old` from the database; /// it will just return the new RLP that includes the new node. /// - /// The database will be updated so as to make the returned RLP valid through inserting + /// `journal` will record the database updates so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. /// /// **This operation will not insert the new node nor destroy the original.** - fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], diff: &mut Diff) -> Bytes { + fn augmented(&self, old: &[u8], partial: &NibbleSlice, value: &[u8], journal: &mut Journal) -> Bytes { trace!("augmented (old: {:?}, partial: {:?}, value: {:?})", old.pretty(), partial, value.pretty()); // already have an extension. either fast_forward, cleve or transmute_to_branch. let old_rlp = Rlp::new(old); @@ -573,7 +592,24 @@ impl TrieDB { Prototype::List(17) => { trace!("branch: ROUTE,AUGMENT"); // already have a branch. route and augment. - self.augmented_into_branch(&old_rlp, partial, value, diff) + let mut s = RlpStream::new_list(17); + let index = if partial.is_empty() {16} else {partial.at(0) as usize}; + for i in 0..17 { + match index == i { + // not us - leave alone. + false => { s.append_raw(old_rlp.at(i).raw(), 1); }, + // branch-leaf entry - just replace. + true if i == 16 => { s.append(&value); }, + // original had empty slot - place a leaf there. + true if old_rlp.at(i).is_empty() => journal.new_node(Self::compose_leaf(&partial.mid(1), value), &mut s), + // original has something there already; augment. + true => { + let new = self.augmented(self.take_node(&old_rlp.at(i), journal), &partial.mid(1), value, journal); + journal.new_node(new, &mut s); + } + } + } + s.out() }, Prototype::List(2) => { let existing_key_rlp = old_rlp.at(0); @@ -587,7 +623,24 @@ impl TrieDB { (_, 0) => { // one of us isn't empty: transmute to branch here trace!("no-common-prefix, not-both-empty (exist={:?}; new={:?}): TRANSMUTE,AUGMENT", existing_key.len(), partial.len()); - self.augmented_into_transmuted_branch(is_leaf, &existing_key, old_rlp.at(1).raw(), partial, value, diff) + assert!(is_leaf || !existing_key.is_empty()); // extension nodes are not allowed to have empty partial keys. + let mut s = RlpStream::new_list(17); + let index = if existing_key.is_empty() {16} else {existing_key.at(0)}; + for i in 0..17 { + match is_leaf { + // not us - empty. + _ if index != i => { s.append_empty_data(); }, + // branch-value: just replace. + true if i == 16 => { s.append_raw(old_rlp.at(1).raw(), 1); }, + // direct extension: just replace. + false if existing_key.len() == 1 => { s.append_raw(old_rlp.at(1).raw(), 1); }, + // original has empty slot. + true => journal.new_node(Self::compose_leaf(&existing_key.mid(1), old_rlp.at(1).data()), &mut s), + // additional work required after branching. + false => journal.new_node(Self::compose_extension(&existing_key.mid(1), old_rlp.at(1).raw()), &mut s), + } + }; + self.augmented(&s.out(), partial, value, journal) }, (_, cp) if cp == existing_key.len() => { trace!("complete-prefix (cp={:?}): AUGMENT-AT-END", cp); @@ -595,10 +648,15 @@ impl TrieDB { // transform to an extension + augmented version of onward node. let downstream_node: Bytes = match is_leaf { // no onward node because we're a leaf - create fake stub and use that. - true => self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, diff), - false => self.augmented(self.take_node(&old_rlp.at(1), diff), &partial.mid(cp), value, diff), + true => self.augmented(&Self::compose_stub_branch(old_rlp.at(1).data()), &partial.mid(cp), value, journal), + false => self.augmented(self.take_node(&old_rlp.at(1), journal), &partial.mid(cp), value, journal), }; - Self::create_extension(&existing_key, downstream_node, diff) + + trace!("create_extension partial: {:?}, downstream_node: {:?}", existing_key, downstream_node.pretty()); + let mut s = RlpStream::new_list(2); + s.append(&existing_key.encoded(false)); + journal.new_node(downstream_node, &mut s); + s.out() }, (_, cp) => { // partially-shared prefix for this extension: @@ -612,12 +670,12 @@ impl TrieDB { // low (farther from root) let low = Self::compose_raw(&existing_key.mid(cp), old_rlp.at(1).raw(), is_leaf); - let augmented_low = self.augmented(&low, &partial.mid(cp), value, diff); + let augmented_low = self.augmented(&low, &partial.mid(cp), value, journal); // high (closer to root) let mut s = RlpStream::new_list(2); s.append(&existing_key.encoded_leftmost(cp, false)); - diff.new_node(augmented_low, &mut s); + journal.new_node(augmented_low, &mut s); s.out() }, } @@ -630,6 +688,7 @@ impl TrieDB { } } + /// Given a `MaybeChanged` result `n`, return the node's RLP regardless of whether it changed. fn encoded(n: MaybeChanged) -> Bytes { match n { MaybeChanged::Same(n) => n.encoded(), @@ -637,18 +696,20 @@ impl TrieDB { } } - fn fixed_indirection<'a>(n: Node<'a>, diff: &mut Diff) -> MaybeChanged<'a> { + /// Fix the node payload's sizes in `n`, replacing any over-size payloads with the hashed reference + /// and placing the payload DB insertions in the `journal`. + fn fixed_indirection<'a>(n: Node<'a>, journal: &mut Journal) -> MaybeChanged<'a> { match n { Node::Extension(partial, payload) if payload.len() >= 32 && Rlp::new(payload).is_list() => { // make indirect - MaybeChanged::Changed(Node::Extension(partial, &Node::decoded(payload).encoded_and_added(diff)).encoded()) + MaybeChanged::Changed(Node::Extension(partial, &Node::decoded(payload).encoded_and_added(journal)).encoded()) }, Node::Branch(payloads, value) => { // check each child isn't too big // TODO OPTIMISE - should really check at the point of (re-)constructing the branch. for i in 0..16 { if payloads[i].len() >= 32 && Rlp::new(payloads[i]).is_list() { - let n = Node::decoded(payloads[i]).encoded_and_added(diff); + let n = Node::decoded(payloads[i]).encoded_and_added(journal); let mut new_nodes = payloads; new_nodes[i] = &n; return MaybeChanged::Changed(Node::Branch(new_nodes, value).encoded()) @@ -668,8 +729,11 @@ impl TrieDB { /// - Extension node followed by anything other than a Branch node. /// - Extension node with a child which has too many bytes to be inline. /// + /// `journal` will record the database updates so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + /// /// **This operation will not insert the new node nor destroy the original.** - fn fixed<'a, 'b>(&'a self, n: Node<'b>, diff: &mut Diff) -> MaybeChanged<'b> where 'a: 'b { + fn fixed<'a, 'b>(&'a self, n: Node<'b>, journal: &mut Journal) -> MaybeChanged<'b> where 'a: 'b { trace!("fixed node={:?}", n); match n { Node::Branch(nodes, node_value) => { @@ -681,9 +745,6 @@ impl TrieDB { Many, }; let mut used_index = UsedIndex::None; - // 0-15 -> index of a non-null branch - // 16 -> no non-null branch - // 17 -> multiple non-null branches for i in 0..16 { match (nodes[i] == NULL_RLP, &used_index) { (false, &UsedIndex::None) => used_index = UsedIndex::One(i as u8), @@ -699,17 +760,17 @@ impl TrieDB { // TODO: OPTIMISE: - don't call fixed again but put the right node in straight away here. // call fixed again since the transmute may cause invalidity. let new_partial: [u8; 1] = [a; 1]; - MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&new_partial[..], 1), nodes[a as usize]), diff))) + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Extension(NibbleSlice::new_offset(&new_partial[..], 1), nodes[a as usize]), journal))) }, (UsedIndex::None, Some(value)) => { // one leaf value // transmute to leaf. // call fixed again since the transmute may cause invalidity. - MaybeChanged::Changed(Self::encoded(self.fixed(Node::Leaf(NibbleSlice::new(&b""[..]), value), diff))) + MaybeChanged::Changed(Self::encoded(self.fixed(Node::Leaf(NibbleSlice::new(&b""[..]), value), journal))) } _ => { // onwards node(s) and/or leaf // no transmute needed, but should still fix the indirection. trace!("no-transmute: FIXINDIRECTION"); - Self::fixed_indirection(Node::Branch(nodes, node_value), diff) + Self::fixed_indirection(Node::Branch(nodes, node_value), journal) }, } }, @@ -717,16 +778,16 @@ impl TrieDB { match Node::decoded(self.get_raw_or_lookup(payload)) { Node::Extension(sub_partial, sub_payload) => { // combine with node below - diff.delete_node_from_slice(payload); - MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Extension(NibbleSlice::new_composed(&partial, &sub_partial), sub_payload), diff))) + journal.delete_node(payload); + MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Extension(NibbleSlice::new_composed(&partial, &sub_partial), sub_payload), journal))) }, Node::Leaf(sub_partial, sub_value) => { // combine with node below - diff.delete_node_from_slice(payload); - MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Leaf(NibbleSlice::new_composed(&partial, &sub_partial), sub_value), diff))) + journal.delete_node(payload); + MaybeChanged::Changed(Self::encoded(Self::fixed_indirection(Node::Leaf(NibbleSlice::new_composed(&partial, &sub_partial), sub_value), journal))) }, // no change, might still have an oversize node inline - fix indirection - _ => Self::fixed_indirection(n, diff), + _ => Self::fixed_indirection(n, journal), } }, // leaf or empty. no change. @@ -739,34 +800,40 @@ impl TrieDB { /// it will just return the new RLP that represents the new node. /// `None` may be returned should no change be needed. /// - /// The database will be updated so as to make the returned RLP valid through inserting + /// `journal` will record the database updates so as to make the returned RLP valid through inserting /// and deleting nodes as necessary. /// /// **This operation will not insert the new node nor destroy the original.** - fn cleared_from_slice(&self, old: &[u8], partial: &NibbleSlice, diff: &mut Diff) -> Option { - self.cleared(Node::decoded(old), partial, diff) + fn cleared_from_slice(&self, old: &[u8], partial: &NibbleSlice, journal: &mut Journal) -> Option { + self.cleared(Node::decoded(old), partial, journal) } - fn cleared(&self, n: Node, partial: &NibbleSlice, diff: &mut Diff) -> Option { + /// Compose the RLP of the node equivalent to `n` except with the `partial` key removed from its (sub-)trie. + /// + /// `journal` will record the database updates so as to make the returned RLP valid through inserting + /// and deleting nodes as necessary. + /// + /// **This operation will not insert the new node nor destroy the original.** + fn cleared(&self, n: Node, partial: &NibbleSlice, journal: &mut Journal) -> Option { trace!("cleared old={:?}, partial={:?})", n, partial); match (n, partial.is_empty()) { (Node::Empty, _) => None, (Node::Branch(_, None), true) => { None }, - (Node::Branch(payloads, _), true) => Some(Self::encoded(self.fixed(Node::Branch(payloads, None), diff))), // matched as leaf-branch - give back fixed branch with it. + (Node::Branch(payloads, _), true) => Some(Self::encoded(self.fixed(Node::Branch(payloads, None), journal))), // matched as leaf-branch - give back fixed branch with it. (Node::Branch(payloads, value), false) => { // Branch with partial left - route, clear, fix. let i: usize = partial.at(0) as usize; trace!("branch-with-partial node[{:?}]={:?}", i, payloads[i].pretty()); - self.cleared(self.get_node(payloads[i]), &partial.mid(1), diff).map(|new_payload| { + self.cleared(self.get_node(payloads[i]), &partial.mid(1), journal).map(|new_payload| { trace!("branch-new-payload={:?}; delete-old={:?}", new_payload.pretty(), payloads[i].pretty()); // downsteam node needed to be changed. - diff.delete_node_from_slice(payloads[i]); + journal.delete_node(payloads[i]); // return fixed up new node. let mut new_payloads = payloads; new_payloads[i] = &new_payload; - Self::encoded(self.fixed(Node::Branch(new_payloads, value), diff)) + Self::encoded(self.fixed(Node::Branch(new_payloads, value), journal)) }) }, (Node::Leaf(node_partial, _), _) => { @@ -785,12 +852,12 @@ impl TrieDB { cp if cp == node_partial.len() => { trace!("matching-prefix (cp={:?}): SKIP,CLEAR,FIXUP", cp); // key at end of extension - skip, clear, fix - self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), diff).map(|new_payload| { + self.cleared(self.get_node(node_payload), &partial.mid(node_partial.len()), journal).map(|new_payload| { trace!("extension-new-payload={:?}; delete-old={:?}", new_payload.pretty(), node_payload.pretty()); // downsteam node needed to be changed. - diff.delete_node_from_slice(node_payload); + journal.delete_node(node_payload); // return fixed up new node. - Self::encoded(self.fixed(Node::Extension(node_partial, &new_payload), diff)) + Self::encoded(self.fixed(Node::Extension(node_partial, &new_payload), journal)) }) }, _ => None, // key in the middle of an extension - doesn't exist. @@ -812,11 +879,23 @@ impl Trie for TrieDB { } fn insert(&mut self, key: &[u8], value: &[u8]) { - self.add(&NibbleSlice::new(key), value); + match value.is_empty() { + false => self.insert_ns(&NibbleSlice::new(key), value), + true => self.remove_ns(&NibbleSlice::new(key)), + } } fn remove(&mut self, key: &[u8]) { - self.delete(&NibbleSlice::new(key)); + self.remove_ns(&NibbleSlice::new(key)); + } +} + +impl fmt::Debug for TrieDB { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(writeln!(f, "c={:?} [", self.hash_count)); + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); + writeln!(f, "]") } } @@ -834,9 +913,9 @@ mod tests { use std::collections::HashSet; use bytes::{ToPretty,Bytes}; - fn random_key(alphabet: &[u8], min_count: usize, diff_count: usize) -> Vec { + fn random_key(alphabet: &[u8], min_count: usize, journal_count: usize) -> Vec { let mut ret: Vec = Vec::new(); - let r = min_count + if diff_count > 0 {random::() % diff_count} else {0}; + let r = min_count + if journal_count > 0 {random::() % journal_count} else {0}; for _ in 0..r { ret.push(alphabet[random::() % alphabet.len()]); } @@ -885,18 +964,18 @@ mod tests { fn playpen() { env_logger::init().ok(); - let maps = map!{ - "six-low" => StandardMap{alphabet: Alphabet::Low, min_key: 6, diff_key: 0, count: 1000}, - "six-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 6, diff_key: 0, count: 1000}, - "six-all" => StandardMap{alphabet: Alphabet::All, min_key: 6, diff_key: 0, count: 1000}, - "mix-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 1, diff_key: 5, count: 1000} + /*let maps = map!{ + "six-low" => StandardMap{alphabet: Alphabet::Low, min_key: 6, journal_key: 0, count: 1000}, + "six-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 6, journal_key: 0, count: 1000}, + "six-all" => StandardMap{alphabet: Alphabet::All, min_key: 6, journal_key: 0, count: 1000}, + "mix-mid" => StandardMap{alphabet: Alphabet::Mid, min_key: 1, journal_key: 5, count: 1000} }; for sm in maps { let m = sm.1.make(); let t = populate_trie(&m); println!("{:?}: root={:?}, hash_count={:?}", sm.0, t.root(), t.hash_count); - }; - panic!(); + };*/ +// panic!(); for test_i in 0..1 { if test_i % 50 == 0 { @@ -1156,7 +1235,7 @@ mod tests { #[test] fn stress() { - for _ in 0..5000 { + for _ in 0..500 { let mut x: Vec<(Vec, Vec)> = Vec::new(); let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; for j in 0..4u32 { From 36bea4692e01dd84cd52d79b20571a9f128f9c02 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 7 Dec 2015 16:32:06 +0100 Subject: [PATCH 230/379] refector init --- benches/rlp.rs | 2 +- src/rlp/faces.rs | 23 ++++++ src/rlp/mod.rs | 7 ++ src/{rlp.rs => rlp/old.rs} | 15 ++-- src/rlp/rlp.rs | 72 +++++++++++++++++++ src/rlp/rlpstream.rs | 0 src/rlp/untrusted_rlp.rs | 143 +++++++++++++++++++++++++++++++++++++ 7 files changed, 256 insertions(+), 6 deletions(-) create mode 100644 src/rlp/faces.rs create mode 100644 src/rlp/mod.rs rename src/{rlp.rs => rlp/old.rs} (99%) create mode 100644 src/rlp/rlp.rs create mode 100644 src/rlp/rlpstream.rs create mode 100644 src/rlp/untrusted_rlp.rs diff --git a/benches/rlp.rs b/benches/rlp.rs index 1160e311f..62f6c6112 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -91,6 +91,6 @@ fn bench_stream_1000_empty_lists(b: &mut Bencher) { for _ in 0..1000 { stream.append_list(0); } - let _ = stream.out(); + //let _ = stream.out(); }); } diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs new file mode 100644 index 000000000..8ebe962c5 --- /dev/null +++ b/src/rlp/faces.rs @@ -0,0 +1,23 @@ +pub trait Reader<'a, 'view>: Sized { + type Prototype; + type PayloadInfo; + type Data; + type Item; + + fn new(bytes: &'a [u8]) -> Self; + fn raw(&'view self) -> &'a [u8]; + fn prototype(&self) -> Self::Prototype; + fn payload_info(&self) -> Self::PayloadInfo; + fn data(&'view self) -> Self::Data; + fn item_count(&self) -> usize; + fn size(&self) -> usize; + fn at(&'view self, index: usize) -> Self::Item; + fn is_null(&self) -> bool; + fn is_empty(&self) -> bool; + fn is_list(&self) -> bool; + fn is_data(&self) -> bool; + fn is_int(&self) -> bool; +} + +pub trait Stream { +} diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs new file mode 100644 index 000000000..98e73665f --- /dev/null +++ b/src/rlp/mod.rs @@ -0,0 +1,7 @@ +pub mod old; + +pub mod faces; +pub mod rlp; +pub mod untrusted_rlp; + +pub use self::old::*; diff --git a/src/rlp.rs b/src/rlp/old.rs similarity index 99% rename from src/rlp.rs rename to src/rlp/old.rs index d0dece3ba..63c3040e0 100644 --- a/src/rlp.rs +++ b/src/rlp/old.rs @@ -32,10 +32,10 @@ use std::fmt; use std::cell::Cell; +use std::ops::Deref; use std::error::Error as StdError; use elastic_array::*; use bytes::{ToBytes, FromBytes, FromBytesError}; -use vector::InsertSlice; /// Data-oriented view onto rlp-slice. /// @@ -113,6 +113,14 @@ pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } +//impl<'a> Deref for Rlp<'a> { + //type Target = UntrustedRlp<'a>; + + //fn deref(&self) -> &Self::Target { + //&self.rlp + //} +//} + impl<'a> From> for Rlp<'a> { fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { Rlp { rlp: rlp } @@ -837,8 +845,6 @@ impl RlpStream { /// } /// ``` pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { - // push new list - let position = self.encoder.bytes.len(); match len { 0 => { // we may finish, if the appended list len is equal 0 @@ -846,8 +852,7 @@ impl RlpStream { self.note_appended(1); }, _ => { - // reserve at least double size of the len - //self.encoder.bytes.reserve(len * 2); + let position = self.encoder.bytes.len(); self.unfinished_lists.push(ListInfo::new(position, len)); }, } diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs new file mode 100644 index 000000000..603ba425d --- /dev/null +++ b/src/rlp/rlp.rs @@ -0,0 +1,72 @@ +use super::faces::Reader; +use super::untrusted_rlp::*; + +/// Data-oriented view onto trusted rlp-slice. +/// +/// Unlikely to `UntrustedRlp` doesn't bother you with error +/// handling. It assumes that you know what you are doing. +pub struct Rlp<'a> { + rlp: UntrustedRlp<'a> +} + +impl<'a, 'view> Reader<'a, 'view> for Rlp<'a> where 'a: 'view { + type Prototype = Prototype; + type PayloadInfo = PayloadInfo; + type Data = &'a [u8]; + type Item = Rlp<'a>; + + /// Create a new instance of `Rlp` + fn new(bytes: &'a [u8]) -> Rlp<'a> { + Rlp { + rlp: UntrustedRlp::new(bytes) + } + } + + fn raw(&'view self) -> &'a [u8] { + self.rlp.raw() + } + + fn prototype(&self) -> Self::Prototype { + unimplemented!() + } + + fn payload_info(&self) -> Self::PayloadInfo { + unimplemented!() + } + + fn data(&'view self) -> Self::Data { + unimplemented!() + } + + fn item_count(&self) -> usize { + unimplemented!() + } + + fn size(&self) -> usize { + unimplemented!() + } + + fn at(&'view self, index: usize) -> Self::Item { + unimplemented!() + } + + fn is_null(&self) -> bool { + unimplemented!() + } + + fn is_empty(&self) -> bool { + unimplemented!() + } + + fn is_list(&self) -> bool { + unimplemented!() + } + + fn is_data(&self) -> bool { + unimplemented!() + } + + fn is_int(&self) -> bool { + unimplemented!() + } +} diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs new file mode 100644 index 000000000..e69de29bb diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs new file mode 100644 index 000000000..8d875324e --- /dev/null +++ b/src/rlp/untrusted_rlp.rs @@ -0,0 +1,143 @@ +use std::fmt; +use std::cell::Cell; +use std::error::Error as StdError; +use bytes::{FromBytesError}; +use super::faces::Reader; + +/// rlp offset +#[derive(Copy, Clone, Debug)] +struct OffsetCache { + index: usize, + offset: usize, +} + +impl OffsetCache { + fn new(index: usize, offset: usize) -> OffsetCache { + OffsetCache { + index: index, + offset: offset, + } + } +} + +#[derive(Debug)] +pub enum Prototype { + Null, + Data(usize), + List(usize), +} + +/// Stores basic information about item +pub struct PayloadInfo { + pub header_len: usize, + pub value_len: usize, +} + +impl PayloadInfo { + fn new(header_len: usize, value_len: usize) -> PayloadInfo { + PayloadInfo { + header_len: header_len, + value_len: value_len, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, +} +impl StdError for DecoderError { + fn description(&self) -> &str { + "builder error" + } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } +} + +/// Data-oriented view onto rlp-slice. +/// +/// This is immutable structere. No operations change it. +/// +/// Should be used in places where, error handling is required, +/// eg. on input +#[derive(Debug)] +pub struct UntrustedRlp<'a> { + bytes: &'a [u8], + cache: Cell, +} + +impl<'a, 'view> Reader<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { + type Prototype = Result; + type PayloadInfo = Result; + type Data = Result<&'a [u8], DecoderError>; + type Item = Result, DecoderError>; + + //returns new instance of `UntrustedRlp` + fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { + UntrustedRlp { + bytes: bytes, + cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), + } + } + + fn raw(&'view self) -> &'a [u8] { + self.bytes + } + + fn prototype(&self) -> Self::Prototype { + unimplemented!() + } + + fn payload_info(&self) -> Self::PayloadInfo { + unimplemented!() + } + + fn data(&'view self) -> Self::Data { + unimplemented!() + } + + fn item_count(&self) -> usize { + unimplemented!() + } + + fn size(&self) -> usize { + unimplemented!() + } + + fn at(&'view self, index: usize) -> Self::Item { + unimplemented!() + } + + fn is_null(&self) -> bool { + unimplemented!() + } + + fn is_empty(&self) -> bool { + unimplemented!() + } + + fn is_list(&self) -> bool { + unimplemented!() + } + + fn is_data(&self) -> bool { + unimplemented!() + } + + fn is_int(&self) -> bool { + unimplemented!() + } +} From 03674cad3be346b00e19641a7c254dbaa15fbc85 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 7 Dec 2015 17:20:15 +0100 Subject: [PATCH 231/379] Documentation. --- src/error.rs | 3 +++ src/hash.rs | 6 +++++- src/hashdb.rs | 2 ++ src/trie.rs | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index f6c64a54f..71514d045 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,3 +1,5 @@ +//! General error types for use in ethcore. + use rustc_serialize::hex::*; #[derive(Debug)] @@ -6,6 +8,7 @@ pub enum BaseDataError { } #[derive(Debug)] +/// General error type which should be capable of representing all errors in ethcore. pub enum EthcoreError { FromHex(FromHexError), BaseData(BaseDataError), diff --git a/src/hash.rs b/src/hash.rs index 5662d713c..b51bbdacf 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,3 +1,5 @@ +//! General hash types, a fixed-size raw-data type used as the output of hash functions. + use std::str::FromStr; use std::fmt; use std::ops; @@ -11,7 +13,9 @@ use bytes::BytesConvertable; use math::log2; use uint::U256; -/// types implementing FixedHash must be also BytesConvertable +/// Trait for a fixed-size byte array to be used as the output of hash functions. +/// +/// Note: types implementing `FixedHash` must be also `BytesConvertable`. pub trait FixedHash: Sized + BytesConvertable { fn new() -> Self; fn random() -> Self; diff --git a/src/hashdb.rs b/src/hashdb.rs index 207883e4b..e63d031e5 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -1,7 +1,9 @@ +//! Database of byte-slices keyed to their Keccak hash. use hash::*; use bytes::*; use std::collections::HashMap; +/// Trait modelling datastore keyed by a 32-byte Keccak hash. pub trait HashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; diff --git a/src/trie.rs b/src/trie.rs index 0425565bc..5c4420787 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -1,3 +1,4 @@ +//! Key-value datastore with a modified Merkle tree. extern crate rand; use std::fmt; From 7e287755ca491bde2c9a27bb91c2bc43f64c1c4a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 7 Dec 2015 23:47:26 +0100 Subject: [PATCH 232/379] rlp view interface + refactor --- src/rlp/faces.rs | 179 +++++++++++++++++++++++++++++- src/rlp/mod.rs | 1 + src/rlp/rlp.rs | 83 +++++++++++--- src/rlp/untrusted_rlp.rs | 228 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 462 insertions(+), 29 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index 8ebe962c5..69e39b06d 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -1,23 +1,198 @@ -pub trait Reader<'a, 'view>: Sized { +pub trait Decoder { + type Error; + + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result; + + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result; +} + +pub trait Decodable: Sized { + fn decode(decoder: &D) -> Result where D: Decoder; +} + +pub trait View<'a, 'view>: Sized { type Prototype; type PayloadInfo; type Data; type Item; + type Iter; + type Error; + /// Creates a new instance of `Rlp` reader fn new(bytes: &'a [u8]) -> Self; + + /// The raw data of the RLP. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let dog = rlp.at(1).raw(); + /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); + /// } + /// ``` fn raw(&'view self) -> &'a [u8]; + + /// Get the prototype of the RLP. fn prototype(&self) -> Self::Prototype; + fn payload_info(&self) -> Self::PayloadInfo; + fn data(&'view self) -> Self::Data; + + /// Returns number of RLP items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.item_count(), 2); + /// let view = rlp.at(1); + /// assert_eq!(view.item_count(), 0); + /// } + /// ``` fn item_count(&self) -> usize; + + /// Returns the number of bytes in the data, or zero if it isn't data. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.size(), 0); + /// let view = rlp.at(1); + /// assert_eq!(view.size(), 3); + /// } + /// ``` fn size(&self) -> usize; + + /// Get view onto RLP-slice at index. + /// + /// Caches offset to given index, so access to successive + /// slices is faster. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let dog = String::decode(&rlp.at(1)); + /// assert_eq!(dog, "dog".to_string()); + /// } fn at(&'view self, index: usize) -> Self::Item; + + /// No value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_null()); + /// } + /// ``` fn is_null(&self) -> bool; + + /// Contains a zero-length string or zero-length list. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc0]; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_empty()); + /// } + /// ``` fn is_empty(&self) -> bool; + + /// List value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.is_list()); + /// } + /// ``` fn is_list(&self) -> bool; + + /// String value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// assert!(rlp.at(1).is_data()); + /// } + /// ``` fn is_data(&self) -> bool; + + /// Int value + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc1, 0x10]; + /// let rlp = Rlp::new(&data); + /// assert_eq!(rlp.is_int(), false); + /// assert_eq!(rlp.at(0).is_int(), true); + /// } + /// ``` fn is_int(&self) -> bool; + + /// Get iterator over rlp-slices + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + /// let rlp = Rlp::new(&data); + /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); + /// } + /// ``` + fn iter(&'view self) -> Self::Iter; + + fn as_val(&self) -> Result where T: Decodable; } -pub trait Stream { +pub trait Encoder { + fn emit_value(&mut self, bytes: &[u8]) -> (); + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); +} + +pub trait Encodable { + fn encode(&self, encoder: &mut E) -> () where E: Encoder; +} + +pub trait Stream: Sized { + fn new() -> Self; + fn new_list(len: usize) -> Self; + fn append<'a, E>(&'a mut self, object: &E) -> &'a mut Self; } diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 98e73665f..af6da14a2 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -1,6 +1,7 @@ pub mod old; pub mod faces; +pub mod coders; pub mod rlp; pub mod untrusted_rlp; diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index 603ba425d..54807bfda 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -1,19 +1,28 @@ -use super::faces::Reader; +use super::faces::{View, Decodable}; use super::untrusted_rlp::*; +impl<'a> From> for Rlp<'a> { + fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { + Rlp { rlp: rlp } + } +} + /// Data-oriented view onto trusted rlp-slice. /// /// Unlikely to `UntrustedRlp` doesn't bother you with error /// handling. It assumes that you know what you are doing. +#[derive(Debug)] pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } -impl<'a, 'view> Reader<'a, 'view> for Rlp<'a> where 'a: 'view { +impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { type Prototype = Prototype; type PayloadInfo = PayloadInfo; type Data = &'a [u8]; type Item = Rlp<'a>; + type Iter = RlpIterator<'a, 'view>; + type Error = DecoderError; /// Create a new instance of `Rlp` fn new(bytes: &'a [u8]) -> Rlp<'a> { @@ -27,46 +36,94 @@ impl<'a, 'view> Reader<'a, 'view> for Rlp<'a> where 'a: 'view { } fn prototype(&self) -> Self::Prototype { - unimplemented!() + self.rlp.prototype().unwrap() } fn payload_info(&self) -> Self::PayloadInfo { - unimplemented!() + self.rlp.payload_info().unwrap() } fn data(&'view self) -> Self::Data { - unimplemented!() + self.rlp.data().unwrap() } fn item_count(&self) -> usize { - unimplemented!() + self.rlp.item_count() } fn size(&self) -> usize { - unimplemented!() + self.rlp.size() } fn at(&'view self, index: usize) -> Self::Item { - unimplemented!() + From::from(self.rlp.at(index).unwrap()) } fn is_null(&self) -> bool { - unimplemented!() + self.rlp.is_null() } fn is_empty(&self) -> bool { - unimplemented!() + self.rlp.is_empty() } fn is_list(&self) -> bool { - unimplemented!() + self.rlp.is_list() } fn is_data(&self) -> bool { - unimplemented!() + self.rlp.is_data() } fn is_int(&self) -> bool { - unimplemented!() + self.rlp.is_int() + } + + fn iter(&'view self) -> Self::Iter { + self.into_iter() + } + + fn as_val(&self) -> Result where T: Decodable { + self.rlp.as_val() + } +} + +impl <'a, 'view> Rlp<'a> where 'a: 'view { + fn reader_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { + let res: Result = r.as_val(); + res.unwrap_or_else(|_| panic!()) + } + + pub fn as_val(&self) -> T where T: Decodable { + Self::reader_as_val(self) + } +} + +/// Iterator over trusted rlp-slice list elements. +pub struct RlpIterator<'a, 'view> where 'a: 'view { + rlp: &'view Rlp<'a>, + index: usize +} + +impl<'a, 'view> IntoIterator for &'view Rlp<'a> where 'a: 'view { + type Item = Rlp<'a>; + type IntoIter = RlpIterator<'a, 'view>; + + fn into_iter(self) -> Self::IntoIter { + RlpIterator { + rlp: self, + index: 0, + } + } +} + +impl<'a, 'view> Iterator for RlpIterator<'a, 'view> { + type Item = Rlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.rlp.at(index).ok().map(| iter | { From::from(iter) }); + self.index += 1; + result } } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 8d875324e..910696565 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,8 +1,8 @@ use std::fmt; use std::cell::Cell; use std::error::Error as StdError; -use bytes::{FromBytesError}; -use super::faces::Reader; +use bytes::{FromBytes, FromBytesError}; +use super::faces::{View, Decoder, Decodable}; /// rlp offset #[derive(Copy, Clone, Debug)] @@ -79,11 +79,22 @@ pub struct UntrustedRlp<'a> { cache: Cell, } -impl<'a, 'view> Reader<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { +impl<'a> Clone for UntrustedRlp<'a> { + fn clone(&self) -> UntrustedRlp<'a> { + UntrustedRlp { + bytes: self.bytes, + cache: Cell::new(OffsetCache::new(usize::max_value(), 0)) + } + } +} + +impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { type Prototype = Result; type PayloadInfo = Result; type Data = Result<&'a [u8], DecoderError>; type Item = Result, DecoderError>; + type Iter = UntrustedRlpIterator<'a, 'view>; + type Error = DecoderError; //returns new instance of `UntrustedRlp` fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { @@ -98,46 +109,235 @@ impl<'a, 'view> Reader<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { } fn prototype(&self) -> Self::Prototype { - unimplemented!() + // optimize? && return appropriate errors + if self.is_data() { + Ok(Prototype::Data(self.size())) + } else if self.is_list() { + Ok(Prototype::List(self.item_count())) + } else { + Ok(Prototype::Null) + } } fn payload_info(&self) -> Self::PayloadInfo { - unimplemented!() + BasicDecoder::payload_info(self.bytes) } fn data(&'view self) -> Self::Data { - unimplemented!() + let pi = try!(BasicDecoder::payload_info(self.bytes)); + Ok(&self.bytes[pi.header_len..(pi.header_len + pi.value_len)]) } fn item_count(&self) -> usize { - unimplemented!() + match self.is_list() { + true => self.iter().count(), + false => 0 + } } fn size(&self) -> usize { - unimplemented!() + match self.is_data() { + // we can safely unwrap (?) cause its data + true => BasicDecoder::payload_info(self.bytes).unwrap().value_len, + false => 0 + } } fn at(&'view self, index: usize) -> Self::Item { - unimplemented!() + if !self.is_list() { + return Err(DecoderError::RlpExpectedToBeList); + } + + // move to cached position if it's index is less or equal to + // current search index, otherwise move to beginning of list + let c = self.cache.get(); + let (mut bytes, to_skip) = match c.index <= index { + true => (try!(UntrustedRlp::consume(self.bytes, c.offset)), index - c.index), + false => (try!(self.consume_list_prefix()), index), + }; + + // skip up to x items + bytes = try!(UntrustedRlp::consume_items(bytes, to_skip)); + + // update the cache + self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); + + // construct new rlp + let found = try!(BasicDecoder::payload_info(bytes)); + Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) } fn is_null(&self) -> bool { - unimplemented!() + self.bytes.len() == 0 } fn is_empty(&self) -> bool { - unimplemented!() + !self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80) } fn is_list(&self) -> bool { - unimplemented!() + !self.is_null() && self.bytes[0] >= 0xc0 } fn is_data(&self) -> bool { - unimplemented!() + !self.is_null() && self.bytes[0] < 0xc0 } fn is_int(&self) -> bool { - unimplemented!() + if self.is_null() { + return false; + } + + match self.bytes[0] { + 0...0x80 => true, + 0x81...0xb7 => self.bytes[1] != 0, + b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0, + _ => false + } + } + + fn iter(&'view self) -> Self::Iter { + self.into_iter() + } + + fn as_val(&self) -> Result where T: Decodable { + // optimize, so it doesn't use clone (although This clone is cheap) + T::decode(&BasicDecoder::new(self.clone())) } } + +impl<'a> UntrustedRlp<'a> { + /// consumes first found prefix + fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { + let item = try!(BasicDecoder::payload_info(self.bytes)); + let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); + Ok(bytes) + } + + /// consumes fixed number of items + fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { + let mut result = bytes; + for _ in 0..items { + let i = try!(BasicDecoder::payload_info(result)); + result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); + } + Ok(result) + } + + + /// consumes slice prefix of length `len` + fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { + match bytes.len() >= len { + true => Ok(&bytes[len..]), + false => Err(DecoderError::RlpIsTooShort), + } + } +} + +/// Iterator over rlp-slice list elements. +pub struct UntrustedRlpIterator<'a, 'view> where 'a: 'view { + rlp: &'view UntrustedRlp<'a>, + index: usize, +} + +impl<'a, 'view> IntoIterator for &'view UntrustedRlp<'a> where 'a: 'view { + type Item = UntrustedRlp<'a>; + type IntoIter = UntrustedRlpIterator<'a, 'view>; + + fn into_iter(self) -> Self::IntoIter { + UntrustedRlpIterator { + rlp: self, + index: 0, + } + } +} + +impl<'a, 'view> Iterator for UntrustedRlpIterator<'a, 'view> { + type Item = UntrustedRlp<'a>; + + fn next(&mut self) -> Option> { + let index = self.index; + let result = self.rlp.at(index).ok(); + self.index += 1; + result + } +} + +struct BasicDecoder<'a> { + rlp: UntrustedRlp<'a> +} + +impl<'a> BasicDecoder<'a> { + pub fn new(rlp: UntrustedRlp<'a>) -> BasicDecoder<'a> { + BasicDecoder { + rlp: rlp + } + } + + /// Return first item info + fn payload_info(bytes: &[u8]) -> Result { + let item = match bytes.first().map(|&x| x) { + None => return Err(DecoderError::RlpIsTooShort), + Some(0...0x7f) => PayloadInfo::new(0, 1), + Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + } + Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), + Some(l @ 0xf8...0xff) => { + let len_of_len = l as usize - 0xf7; + let header_len = 1 + len_of_len; + let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + PayloadInfo::new(header_len, value_len) + }, + // we cant reach this place, but rust requires _ to be implemented + _ => { panic!(); } + }; + + match item.header_len + item.value_len <= bytes.len() { + true => Ok(item), + false => Err(DecoderError::RlpIsTooShort), + } + } +} + +impl<'a> Decoder for BasicDecoder<'a> { + type Error = DecoderError; + + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result { + + let bytes = self.rlp.raw(); + + match bytes.first().map(|&x| x) { + // rlp is too short + None => Err(DecoderError::RlpIsTooShort), + // single byt value + Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), + // 0-55 bytes + Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), + // longer than 55 bytes + Some(l @ 0xb8...0xbf) => { + let len_of_len = l as usize - 0xb7; + let begin_of_value = 1 as usize + len_of_len; + let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); + Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) + } + // we are reading value, not a list! + _ => { unreachable!(); } + } + } + + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result { + + let v: Vec> = self.rlp.iter() + .map(| i | BasicDecoder::new(i)) + .collect(); + f(&v) + } +} + From 0de75185aad990fcd8a43a69e59e527cc9190807 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:07:07 +0100 Subject: [PATCH 233/379] decodable trait impl in progress --- src/rlp/faces.rs | 45 +++++++++++++++++++++++------ src/rlp/mod.rs | 34 +++++++++++++++++++++- src/rlp/rlp.rs | 7 ++--- src/rlp/untrusted_rlp.rs | 61 ++++++++++++++++------------------------ 4 files changed, 96 insertions(+), 51 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index 69e39b06d..d72f2f96c 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -1,15 +1,43 @@ +use std::fmt; +use std::error::Error as StdError; +use bytes::FromBytesError; + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, +} + +impl StdError for DecoderError { + fn description(&self) -> &str { + "builder error" + } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } +} + pub trait Decoder { - type Error; + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result; - fn read_value(&self, f: F) -> Result - where F: FnOnce(&[u8]) -> Result; - - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result; + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result; } pub trait Decodable: Sized { - fn decode(decoder: &D) -> Result where D: Decoder; + fn decode(decoder: &D) -> Result where D: Decoder; } pub trait View<'a, 'view>: Sized { @@ -18,7 +46,6 @@ pub trait View<'a, 'view>: Sized { type Data; type Item; type Iter; - type Error; /// Creates a new instance of `Rlp` reader fn new(bytes: &'a [u8]) -> Self; @@ -179,7 +206,7 @@ pub trait View<'a, 'view>: Sized { /// ``` fn iter(&'view self) -> Self::Iter; - fn as_val(&self) -> Result where T: Decodable; + fn as_val(&self) -> Result where T: Decodable; } pub trait Encoder { diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index af6da14a2..98cb43e80 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -1,8 +1,40 @@ +//! Rlp serialization module +//! +//! Allows encoding, decoding, and view onto rlp-slice +//! +//!# What should you use when? +//! +//!### Use `encode` function when: +//! * You want to encode something inline. +//! * You do not work on big set of data. +//! * You want to encode whole data structure at once. +//! +//!### Use `decode` function when: +//! * You want to decode something inline. +//! * You do not work on big set of data. +//! * You want to decode whole rlp at once. +//! +//!### Use `RlpStream` when: +//! * You want to encode something in portions. +//! * You encode a big set of data. +//! +//!### Use `Rlp` when: +//! * You are working on trusted data (not corrupted). +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. +//! +//!### Use `UntrustedRlp` when: +//! * You are working on untrusted data (~corrupted). +//! * You need to handle data corruption errors. +//! * You are working on input data. +//! * You want to get view onto rlp-slice. +//! * You don't want to decode whole rlp at once. + pub mod old; pub mod faces; -pub mod coders; pub mod rlp; pub mod untrusted_rlp; +pub mod rlpstream; pub use self::old::*; diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index 54807bfda..2effb7297 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -1,4 +1,4 @@ -use super::faces::{View, Decodable}; +use super::faces::{View, Decodable, DecoderError}; use super::untrusted_rlp::*; impl<'a> From> for Rlp<'a> { @@ -22,7 +22,6 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { type Data = &'a [u8]; type Item = Rlp<'a>; type Iter = RlpIterator<'a, 'view>; - type Error = DecoderError; /// Create a new instance of `Rlp` fn new(bytes: &'a [u8]) -> Rlp<'a> { @@ -83,14 +82,14 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { self.into_iter() } - fn as_val(&self) -> Result where T: Decodable { + fn as_val(&self) -> Result where T: Decodable { self.rlp.as_val() } } impl <'a, 'view> Rlp<'a> where 'a: 'view { fn reader_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { - let res: Result = r.as_val(); + let res: Result = r.as_val(); res.unwrap_or_else(|_| panic!()) } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 910696565..89c3b664f 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,8 +1,6 @@ -use std::fmt; use std::cell::Cell; -use std::error::Error as StdError; -use bytes::{FromBytes, FromBytesError}; -use super::faces::{View, Decoder, Decodable}; +use bytes::{FromBytes}; +use super::faces::{View, Decoder, Decodable, DecoderError}; /// rlp offset #[derive(Copy, Clone, Debug)] @@ -42,31 +40,6 @@ impl PayloadInfo { } } -#[derive(Debug, PartialEq, Eq)] -pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeData, -} -impl StdError for DecoderError { - fn description(&self) -> &str { - "builder error" - } -} - -impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { - DecoderError::FromBytesError(err) - } -} - /// Data-oriented view onto rlp-slice. /// /// This is immutable structere. No operations change it. @@ -94,7 +67,6 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { type Data = Result<&'a [u8], DecoderError>; type Item = Result, DecoderError>; type Iter = UntrustedRlpIterator<'a, 'view>; - type Error = DecoderError; //returns new instance of `UntrustedRlp` fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { @@ -200,7 +172,7 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { self.into_iter() } - fn as_val(&self) -> Result where T: Decodable { + fn as_val(&self) -> Result where T: Decodable { // optimize, so it doesn't use clone (although This clone is cheap) T::decode(&BasicDecoder::new(self.clone())) } @@ -305,10 +277,8 @@ impl<'a> BasicDecoder<'a> { } impl<'a> Decoder for BasicDecoder<'a> { - type Error = DecoderError; - - fn read_value(&self, f: F) -> Result - where F: FnOnce(&[u8]) -> Result { + fn read_value(&self, f: F) -> Result + where F: FnOnce(&[u8]) -> Result { let bytes = self.rlp.raw(); @@ -331,8 +301,8 @@ impl<'a> Decoder for BasicDecoder<'a> { } } - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result { + fn read_list(&self, f: F) -> Result + where F: FnOnce(&[Self]) -> Result { let v: Vec> = self.rlp.iter() .map(| i | BasicDecoder::new(i)) @@ -341,3 +311,20 @@ impl<'a> Decoder for BasicDecoder<'a> { } } +impl Decodable for T where T: FromBytes { + fn decode(decoder: &D) -> Result where D: Decoder { + unimplemented!() + } +} + +impl Decodable for Vec where T: Decodable { + fn decode(decoder: &D) -> Result where D: Decoder { + unimplemented!() + } +} + +impl Decodable for Vec { + fn decode(decoder: &D) -> Result where D: Decoder { + unimplemented!() + } +} From cb5ec8c0afe77c6221d0c6facb01bd8086739661 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:27:12 +0100 Subject: [PATCH 234/379] decodable implementations --- src/rlp/faces.rs | 2 +- src/rlp/untrusted_rlp.rs | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index d72f2f96c..392908f16 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -37,7 +37,7 @@ pub trait Decoder { } pub trait Decodable: Sized { - fn decode(decoder: &D) -> Result where D: Decoder; + fn decode(decoder: &D) -> Result where D: Decoder; } pub trait View<'a, 'view>: Sized { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 89c3b664f..fd385b99d 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -312,19 +312,27 @@ impl<'a> Decoder for BasicDecoder<'a> { } impl Decodable for T where T: FromBytes { - fn decode(decoder: &D) -> Result where D: Decoder { - unimplemented!() + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_value(| bytes | { + Ok(try!(T::from_bytes(bytes))) + }) } } impl Decodable for Vec where T: Decodable { - fn decode(decoder: &D) -> Result where D: Decoder { - unimplemented!() + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_list(| decoders | { + decoders.iter().map(|d| T::decode(d)).collect() + }) } } impl Decodable for Vec { - fn decode(decoder: &D) -> Result where D: Decoder { - unimplemented!() + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_value(| bytes | { + let mut res = vec![]; + res.extend(bytes); + Ok(res) + }) } } From 4419924154fdf3edf7efed68401a9620e38ca02b Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:45:04 +0100 Subject: [PATCH 235/379] decoder refactor applied --- src/overlaydb.rs | 2 +- src/rlp/mod.rs | 12 +++++++++++- src/rlp/old.rs | 26 +++++++++++++++----------- src/trie.rs | 8 ++++---- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/overlaydb.rs b/src/overlaydb.rs index c13acfd6a..b5bbf4299 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -122,7 +122,7 @@ impl OverlayDB { .expect("Low-level database error. Some issue with your hard disk?") .map(|d| { let r = Rlp::new(d.deref()); - (Bytes::decode(&r.at(1)), u32::decode(&r.at(0))) + (r.at(1).as_val(), r.at(0).as_val()) }) } diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 98cb43e80..e14b4ad25 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -37,4 +37,14 @@ pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; -pub use self::old::*; +pub use self::faces::{DecoderError, Decoder, Decodable, View}; +pub use self::rlp::*; +pub use self::untrusted_rlp::*; + +pub use self::old::{encode, RlpStream, Encodable}; +//pub use self::old::*; + +pub fn decode(bytes: &[u8]) -> T where T: Decodable { + let rlp = Rlp::new(bytes); + rlp.as_val() +} diff --git a/src/rlp/old.rs b/src/rlp/old.rs index 63c3040e0..dd09ea548 100644 --- a/src/rlp/old.rs +++ b/src/rlp/old.rs @@ -1119,7 +1119,7 @@ mod tests { use std::{fmt, cmp}; use std::str::FromStr; use rlp; - use rlp::{UntrustedRlp, RlpStream, Decodable}; + use rlp::{UntrustedRlp, RlpStream, Decodable, View}; use uint::U256; #[test] @@ -1128,23 +1128,27 @@ mod tests { { let rlp = UntrustedRlp::new(&data); assert!(rlp.is_list()); - let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); + //let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); + let animals: Vec = rlp.as_val().unwrap(); assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); let cat = rlp.at(0).unwrap(); assert!(cat.is_data()); - assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); + assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); assert!(dog.is_data()); - assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); - assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); + assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_data()); - assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); - assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); } } @@ -1172,18 +1176,18 @@ mod tests { let cat = iter.next().unwrap(); assert!(cat.is_data()); - assert_eq!(cat.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); let dog = iter.next().unwrap(); assert!(dog.is_data()); - assert_eq!(dog.bytes, &[0x83, b'd', b'o', b'g']); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); let none = iter.next(); assert!(none.is_none()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_data()); - assert_eq!(cat_again.bytes, &[0x83, b'c', b'a', b't']); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); } } diff --git a/src/trie.rs b/src/trie.rs index a340d78f7..0cbfca189 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -131,7 +131,7 @@ impl Diff { fn delete_node(&mut self, old: &Rlp) { if old.is_data() && old.size() == 32 { - self.delete_node_sha3(H256::decode(old)); + self.delete_node_sha3(old.as_val()); } } @@ -310,7 +310,7 @@ impl TrieDB { let mut handle_payload = |payload| { let p = Rlp::new(payload); if p.is_data() && p.size() == 32 { - acc.push(H256::decode(&p)); + acc.push(p.as_val()); } self.accumulate_keys(self.get_node(payload), acc); @@ -417,7 +417,7 @@ impl TrieDB { // check if its sha3 + len let r = Rlp::new(node); match r.is_data() && r.size() == 32 { - true => self.db.lookup(&H256::decode(&r)).expect("Not found!"), + true => self.db.lookup(&r.as_val::()).expect("Not found!"), false => node } } @@ -495,7 +495,7 @@ impl TrieDB { rlp.raw() } else if rlp.is_data() && rlp.size() == 32 { - let h = H256::decode(rlp); + let h = rlp.as_val(); let r = self.db.lookup(&h).unwrap_or_else(||{ println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); println!("Diff: {:?}", diff); From 701aaf126d2977802ec46d15b85d74668c669800 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 00:53:28 +0100 Subject: [PATCH 236/379] fixed rlp reexports --- src/rlp/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index e14b4ad25..a790a480c 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -38,11 +38,10 @@ pub mod untrusted_rlp; pub mod rlpstream; pub use self::faces::{DecoderError, Decoder, Decodable, View}; -pub use self::rlp::*; -pub use self::untrusted_rlp::*; +pub use self::rlp::{Rlp, RlpIterator}; +pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; pub use self::old::{encode, RlpStream, Encodable}; -//pub use self::old::*; pub fn decode(bytes: &[u8]) -> T where T: Decodable { let rlp = Rlp::new(bytes); From 51eff239fa3c6416aaf61e297605744ae0b26d97 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 12:33:05 +0100 Subject: [PATCH 237/379] rlpstream separated to its own submodule --- src/rlp/faces.rs | 92 ++++++++++++++- src/rlp/mod.rs | 36 +++++- src/rlp/rlpstream.rs | 247 +++++++++++++++++++++++++++++++++++++++ src/rlp/untrusted_rlp.rs | 4 +- src/triehash.rs | 2 +- 5 files changed, 375 insertions(+), 6 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index 392908f16..f55b5584e 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -219,7 +219,97 @@ pub trait Encodable { } pub trait Stream: Sized { + + /// Initializes instance of empty `Stream`. fn new() -> Self; + + /// Initializes the `Stream` as a list. fn new_list(len: usize) -> Self; - fn append<'a, E>(&'a mut self, object: &E) -> &'a mut Self; + + /// Apends value to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat").append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + /// ``` + fn append<'a, E>(&'a mut self, object: &E) -> &'a mut Self where E: Encodable; + + /// Declare appending the list of given size, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_list(2).append(&"cat").append(&"dog"); + /// stream.append(&""); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); + /// } + /// ``` + fn append_list<'a>(&'a mut self, len: usize) -> &'a mut Self; + + /// Apends null to the end of stream, chainable. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append_empty_data().append_empty_data(); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); + /// } + /// ``` + fn append_empty_data<'a>(&'a mut self) -> &'a mut Self; + + /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; + + /// Clear the output stream so far. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(3); + /// stream.append(&"cat"); + /// stream.clear(); + /// stream.append(&"dog"); + /// let out = stream.out(); + /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); + /// } + fn clear(&mut self); + + /// Returns true if stream doesnt expect any more items. + /// + /// ```rust + /// extern crate ethcore_util as util; + /// use util::rlp::*; + /// + /// fn main () { + /// let mut stream = RlpStream::new_list(2); + /// stream.append(&"cat"); + /// assert_eq!(stream.is_finished(), false); + /// stream.append(&"dog"); + /// assert_eq!(stream.is_finished(), true); + /// let out = stream.out(); + /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); + /// } + fn is_finished(&self) -> bool; + + /// Streams out encoded bytes. + /// + /// panic! if stream is not finished. + fn out(self) -> Vec; } diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index a790a480c..c7d657fa0 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -37,13 +37,45 @@ pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; -pub use self::faces::{DecoderError, Decoder, Decodable, View}; +pub use self::faces::{DecoderError, Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::rlp::{Rlp, RlpIterator}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; +pub use self::rlpstream::{RlpStream}; -pub use self::old::{encode, RlpStream, Encodable}; +//pub use self::old::{encode, RlpStream, Encodable}; +/// Shortcut function to decode trusted rlp +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; +/// let animals: Vec = decode(&data); +/// assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); +/// } +/// ``` pub fn decode(bytes: &[u8]) -> T where T: Decodable { let rlp = Rlp::new(bytes); rlp.as_val() } + +/// Shortcut function to encode structure into rlp. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::rlp::*; +/// +/// fn main () { +/// let animals = vec!["cat", "dog"]; +/// let out = encode(&animals); +/// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); +/// } +/// ``` +pub fn encode(object: &E) -> Vec where E: Encodable +{ + let mut stream = RlpStream::new(); + stream.append(object); + stream.out() +} diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index e69de29bb..379852083 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -0,0 +1,247 @@ +use elastic_array::*; +use bytes::ToBytes; +use super::faces::{Stream, Encoder, Encodable}; + +#[derive(Debug, Copy, Clone)] +struct ListInfo { + position: usize, + current: usize, + max: usize, +} + +impl ListInfo { + fn new(position: usize, max: usize) -> ListInfo { + ListInfo { + position: position, + current: 0, + max: max, + } + } +} + +/// Appendable rlp encoder. +pub struct RlpStream { + unfinished_lists: ElasticArray16, + encoder: BasicEncoder, +} + +impl Stream for RlpStream { + fn new() -> Self { + RlpStream { + unfinished_lists: ElasticArray16::new(), + encoder: BasicEncoder::new(), + } + } + + fn new_list(len: usize) -> Self { + let mut stream = RlpStream::new(); + stream.append_list(len); + stream + } + + fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { + // encode given value and add it at the end of the stream + object.encode(&mut self.encoder); + + // if list is finished, prepend the length + self.note_appended(1); + + // return chainable self + self + } + + fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { + match len { + 0 => { + // we may finish, if the appended list len is equal 0 + self.encoder.bytes.push(0xc0u8); + self.note_appended(1); + }, + _ => { + let position = self.encoder.bytes.len(); + self.unfinished_lists.push(ListInfo::new(position, len)); + }, + } + + // return chainable self + self + } + + fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { + // self push raw item + self.encoder.bytes.push(0x80); + + // try to finish and prepend the length + self.note_appended(1); + + // return chainable self + self + } + + fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { + // push raw items + self.encoder.bytes.append_slice(bytes); + + // try to finish and prepend the length + self.note_appended(item_count); + + // return chainable self + self + } + + fn clear(&mut self) { + // clear bytes + self.encoder.bytes.clear(); + + // clear lists + self.unfinished_lists.clear(); + } + + fn is_finished(&self) -> bool { + self.unfinished_lists.len() == 0 + } + + fn out(self) -> Vec { + match self.is_finished() { + true => self.encoder.out().to_vec(), + false => panic!() + } + } +} + +impl RlpStream { + + /// Try to finish lists + fn note_appended(&mut self, inserted_items: usize) -> () { + if self.unfinished_lists.len() == 0 { + return; + } + + let back = self.unfinished_lists.len() - 1; + let should_finish = match self.unfinished_lists.get_mut(back) { + None => false, + Some(ref mut x) => { + x.current += inserted_items; + if x.current > x.max { + panic!("You cannot append more items then you expect!"); + } + x.current == x.max + } + }; + + if should_finish { + let x = self.unfinished_lists.pop().unwrap(); + let len = self.encoder.bytes.len() - x.position; + self.encoder.insert_list_len_at_pos(len, x.position); + self.note_appended(1); + } + } +} + +struct BasicEncoder { + bytes: ElasticArray1024, +} + +impl BasicEncoder { + fn new() -> BasicEncoder { + BasicEncoder { bytes: ElasticArray1024::new() } + } + + /// inserts list prefix at given position + /// TODO: optimise it further? + fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { + let mut res = vec![]; + match len { + 0...55 => res.push(0xc0u8 + len as u8), + _ => { + res.push(0xf7u8 + len.to_bytes_len() as u8); + res.extend(len.to_bytes()); + } + }; + + self.bytes.insert_slice(pos, &res); + } + + /// get encoded value + fn out(self) -> ElasticArray1024 { + self.bytes + } +} + +impl Encoder for BasicEncoder { + fn emit_value(&mut self, bytes: &[u8]) -> () { + match bytes.len() { + // just 0 + 0 => self.bytes.push(0x80u8), + // byte is its own encoding + 1 if bytes[0] < 0x80 => self.bytes.append_slice(bytes), + // (prefix + length), followed by the string + len @ 1 ... 55 => { + self.bytes.push(0x80u8 + len as u8); + self.bytes.append_slice(bytes); + } + // (prefix + length of length), followed by the length, followd by the string + len => { + self.bytes.push(0xb7 + len.to_bytes_len() as u8); + self.bytes.append_slice(&len.to_bytes()); + self.bytes.append_slice(bytes); + } + } + } + + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () + { + // get len before inserting a list + let before_len = self.bytes.len(); + + // insert all list elements + f(self); + + // get len after inserting a list + let after_len = self.bytes.len(); + + // diff is list len + let list_len = after_len - before_len; + self.insert_list_len_at_pos(list_len, before_len); + } +} + +impl Encodable for T where T: ToBytes { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(&self.to_bytes()) + } +} + +impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_list(|e| { + // insert all list elements + for el in self.iter() { + el.encode(e); + } + }) + } +} + +impl Encodable for Vec where T: Encodable { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + let r: &[T] = self.as_ref(); + r.encode(encoder) + } +} + +/// lets treat bytes differently than other lists +/// they are a single value +impl<'a> Encodable for &'a [u8] { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} + +/// lets treat bytes differently than other lists +/// they are a single value +impl Encodable for Vec { + fn encode(&self, encoder: &mut E) -> () where E: Encoder { + encoder.emit_value(self) + } +} diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index fd385b99d..9e2e2f522 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -266,7 +266,7 @@ impl<'a> BasicDecoder<'a> { PayloadInfo::new(header_len, value_len) }, // we cant reach this place, but rust requires _ to be implemented - _ => { panic!(); } + _ => { unreachable!(); } }; match item.header_len + item.value_len <= bytes.len() { @@ -297,7 +297,7 @@ impl<'a> Decoder for BasicDecoder<'a> { Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) } // we are reading value, not a list! - _ => { unreachable!(); } + _ => Err(DecoderError::RlpExpectedToBeData) } } diff --git a/src/triehash.rs b/src/triehash.rs index eea79e9d4..67ffcc666 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -7,7 +7,7 @@ use std::cmp; use hash::*; use sha3::*; use rlp; -use rlp::RlpStream; +use rlp::{RlpStream, Stream}; use vector::SharedPrefix; /// Generates a trie root hash for a vector of values From db762c534eae97ddd199f3080d0e7382ce11fd2a Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 12:40:11 +0100 Subject: [PATCH 238/379] docs tests are passing --- src/rlp/faces.rs | 4 ++-- src/rlp/old.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rlp/faces.rs b/src/rlp/faces.rs index f55b5584e..078620de2 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/faces.rs @@ -116,7 +116,7 @@ pub trait View<'a, 'view>: Sized { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// let dog = String::decode(&rlp.at(1)); + /// let dog: String = rlp.at(1).as_val(); /// assert_eq!(dog, "dog".to_string()); /// } fn at(&'view self, index: usize) -> Self::Item; @@ -201,7 +201,7 @@ pub trait View<'a, 'view>: Sized { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); + /// let strings: Vec = rlp.iter().map(| i | i.as_val()).collect(); /// } /// ``` fn iter(&'view self) -> Self::Iter; diff --git a/src/rlp/old.rs b/src/rlp/old.rs index dd09ea548..1c4147463 100644 --- a/src/rlp/old.rs +++ b/src/rlp/old.rs @@ -1119,7 +1119,7 @@ mod tests { use std::{fmt, cmp}; use std::str::FromStr; use rlp; - use rlp::{UntrustedRlp, RlpStream, Decodable, View}; + use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; use uint::U256; #[test] From 36c586eb217f36dd77524c2ebd18da8742e86b15 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 12:59:19 +0100 Subject: [PATCH 239/379] final changes in rlp refactor --- src/rlp/errors.rs | 29 + src/rlp/mod.rs | 13 +- src/rlp/old.rs | 1461 ------------------------------- src/rlp/rlp.rs | 3 +- src/rlp/rlpstream.rs | 2 +- src/rlp/tests.rs | 345 ++++++++ src/rlp/{faces.rs => traits.rs} | 30 +- src/rlp/untrusted_rlp.rs | 2 +- 8 files changed, 385 insertions(+), 1500 deletions(-) create mode 100644 src/rlp/errors.rs delete mode 100644 src/rlp/old.rs create mode 100644 src/rlp/tests.rs rename src/rlp/{faces.rs => traits.rs} (92%) diff --git a/src/rlp/errors.rs b/src/rlp/errors.rs new file mode 100644 index 000000000..ada3c2a47 --- /dev/null +++ b/src/rlp/errors.rs @@ -0,0 +1,29 @@ +use std::fmt; +use std::error::Error as StdError; +use bytes::FromBytesError; + +#[derive(Debug, PartialEq, Eq)] +pub enum DecoderError { + FromBytesError(FromBytesError), + RlpIsTooShort, + RlpExpectedToBeList, + RlpExpectedToBeData, +} + +impl StdError for DecoderError { + fn description(&self) -> &str { + "builder error" + } +} + +impl fmt::Display for DecoderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::Debug::fmt(&self, f) + } +} + +impl From for DecoderError { + fn from(err: FromBytesError) -> DecoderError { + DecoderError::FromBytesError(err) + } +} diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index c7d657fa0..49043d65f 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -30,20 +30,21 @@ //! * You want to get view onto rlp-slice. //! * You don't want to decode whole rlp at once. -pub mod old; - -pub mod faces; +pub mod errors; +pub mod traits; pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; -pub use self::faces::{DecoderError, Decoder, Decodable, View, Stream, Encodable, Encoder}; +#[cfg(test)] +mod tests; + +pub use self::errors::DecoderError; +pub use self::traits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::rlp::{Rlp, RlpIterator}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; pub use self::rlpstream::{RlpStream}; -//pub use self::old::{encode, RlpStream, Encodable}; - /// Shortcut function to decode trusted rlp /// /// ```rust diff --git a/src/rlp/old.rs b/src/rlp/old.rs deleted file mode 100644 index 1c4147463..000000000 --- a/src/rlp/old.rs +++ /dev/null @@ -1,1461 +0,0 @@ -//! Rlp serialization module -//! -//! Allows encoding, decoding, and view onto rlp-slice -//! -//!# What should you use when? -//! -//!### Use `encode` function when: -//! * You want to encode something inline. -//! * You do not work on big set of data. -//! * You want to encode whole data structure at once. -//! -//!### Use `decode` function when: -//! * You want to decode something inline. -//! * You do not work on big set of data. -//! * You want to decode whole rlp at once. -//! -//!### Use `RlpStream` when: -//! * You want to encode something in portions. -//! * You encode a big set of data. -//! -//!### Use `Rlp` when: -//! * You are working on trusted data (not corrupted). -//! * You want to get view onto rlp-slice. -//! * You don't want to decode whole rlp at once. -//! -//!### Use `UntrustedRlp` when: -//! * You are working on untrusted data (~corrupted). -//! * You need to handle data corruption errors. -//! * You are working on input data. -//! * You want to get view onto rlp-slice. -//! * You don't want to decode whole rlp at once. - -use std::fmt; -use std::cell::Cell; -use std::ops::Deref; -use std::error::Error as StdError; -use elastic_array::*; -use bytes::{ToBytes, FromBytes, FromBytesError}; - -/// Data-oriented view onto rlp-slice. -/// -/// This is immutable structere. No operations change it. -/// -/// Should be used in places where, error handling is required, -/// eg. on input -#[derive(Debug)] -pub struct UntrustedRlp<'a> { - bytes: &'a [u8], - cache: Cell, -} - -/// rlp offset -#[derive(Copy, Clone, Debug)] -struct OffsetCache { - index: usize, - offset: usize, -} - -impl OffsetCache { - fn new(index: usize, offset: usize) -> OffsetCache { - OffsetCache { - index: index, - offset: offset, - } - } -} - -/// Stores basic information about item -pub struct PayloadInfo { - pub header_len: usize, - pub value_len: usize, -} - -impl PayloadInfo { - fn new(header_len: usize, value_len: usize) -> PayloadInfo { - PayloadInfo { - header_len: header_len, - value_len: value_len, - } - } -} - -#[derive(Debug, PartialEq, Eq)] -pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeData, -} -impl StdError for DecoderError { - fn description(&self) -> &str { - "builder error" - } -} - -impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { - DecoderError::FromBytesError(err) - } -} - -/// Data-oriented view onto trusted rlp-slice. -/// -/// Unlikely to `UntrustedRlp` doesn't bother you with error -/// handling. It assumes that you know what you are doing. -pub struct Rlp<'a> { - rlp: UntrustedRlp<'a> -} - -//impl<'a> Deref for Rlp<'a> { - //type Target = UntrustedRlp<'a>; - - //fn deref(&self) -> &Self::Target { - //&self.rlp - //} -//} - -impl<'a> From> for Rlp<'a> { - fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { - Rlp { rlp: rlp } - } -} - -impl<'a> From> for UntrustedRlp<'a> { - fn from(unsafe_rlp: Rlp<'a>) -> UntrustedRlp<'a> { - unsafe_rlp.rlp - } -} - -#[derive(Debug)] -pub enum Prototype { - Null, - Data(usize), - List(usize), -} - -impl<'a, 'view> Rlp<'a> where 'a: 'view { - /// Create a new instance of `Rlp` - pub fn new(bytes: &'a [u8]) -> Rlp<'a> { - Rlp { - rlp: UntrustedRlp::new(bytes) - } - } - - /// Get the prototype of the RLP. - pub fn prototype(&self) -> Prototype { - if self.is_data() { - Prototype::Data(self.size()) - } - else if self.is_list() { - Prototype::List(self.item_count()) - } - else { - Prototype::Null - } - } - - /// The raw data of the RLP. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// let dog = rlp.at(1).raw(); - /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); - /// } - /// ``` - pub fn raw(&'view self) -> &'a [u8] { - self.rlp.raw() - } - - pub fn payload_info(&self) -> PayloadInfo { - self.rlp.payload_info().unwrap() - } - - pub fn data(&'view self) -> &'a [u8] { - self.rlp.data() - } - - /// Returns number of RLP items. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.item_count(), 2); - /// let view = rlp.at(1); - /// assert_eq!(view.item_count(), 0); - /// } - /// ``` - pub fn item_count(&self) -> usize { - self.rlp.item_count() - } - - /// Returns the number of bytes in the data, or zero if it isn't data. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.size(), 0); - /// let view = rlp.at(1); - /// assert_eq!(view.size(), 3); - /// } - /// ``` - pub fn size(&self) -> usize { - self.rlp.size() - } - - /// Get view onto RLP-slice at index. - /// - /// Caches offset to given index, so access to successive - /// slices is faster. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// let dog = String::decode(&rlp.at(1)); - /// assert_eq!(dog, "dog".to_string()); - /// } - /// ``` - pub fn at(&'view self, index: usize) -> Rlp<'a> { - From::from(self.rlp.at(index).unwrap()) - } - - /// No value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![]; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.is_null()); - /// } - /// ``` - pub fn is_null(&self) -> bool { - self.rlp.is_null() - } - - /// Contains a zero-length string or zero-length list. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc0]; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.is_empty()); - /// } - /// ``` - pub fn is_empty(&self) -> bool { - self.rlp.is_empty() - } - - /// List value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.is_list()); - /// } - /// ``` - pub fn is_list(&self) -> bool { - self.rlp.is_list() - } - - /// String value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// assert!(rlp.at(1).is_data()); - /// } - /// ``` - pub fn is_data(&self) -> bool { - self.rlp.is_data() - } - - /// Int value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc1, 0x10]; - /// let rlp = Rlp::new(&data); - /// assert_eq!(rlp.is_int(), false); - /// assert_eq!(rlp.at(0).is_int(), true); - /// } - /// ``` - pub fn is_int(&self) -> bool { - self.rlp.is_int() - } - - /// Get iterator over rlp-slices - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = Rlp::new(&data); - /// let strings: Vec = rlp.iter().map(| i | String::decode(&i)).collect(); - /// } - /// ``` - pub fn iter(&'a self) -> RlpIterator<'a> { - self.into_iter() - } -} - -impl<'a, 'view> UntrustedRlp<'a> where 'a: 'view { - /// returns new instance of `UntrustedRlp` - pub fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { - UntrustedRlp { - bytes: bytes, - cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), - } - } - - /// The bare data of the RLP. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// let dog = rlp.at(1).unwrap().raw(); - /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); - /// } - /// ``` - pub fn raw(&'view self) -> &'a [u8] { - self.bytes - } - - pub fn payload_info(&self) -> Result { - BasicDecoder::payload_info(self.bytes) - } - - pub fn data(&'view self) -> &'a [u8] { - let ii = BasicDecoder::payload_info(self.bytes).unwrap(); - &self.bytes[ii.header_len..(ii.header_len + ii.value_len)] - } - - /// Returns number of rlp items. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.item_count(), 2); - /// let view = rlp.at(1).unwrap(); - /// assert_eq!(view.item_count(), 0); - /// } - /// ``` - pub fn item_count(&self) -> usize { - match self.is_list() { - true => self.iter().count(), - false => 0 - } - } - - /// Returns the number of bytes in the data, or zero if it isn't data. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.size(), 0); - /// let view = rlp.at(1).unwrap(); - /// assert_eq!(view.size(), 3); - /// } - /// ``` - pub fn size(&self) -> usize { - match self.is_data() { - true => BasicDecoder::payload_info(self.bytes).unwrap().value_len, - false => 0 - } - } - - /// Get view onto rlp-slice at index - /// - /// Caches offset to given index, so access to successive - /// slices is faster. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// let dog = String::decode_untrusted(&rlp.at(1).unwrap()).unwrap(); - /// assert_eq!(dog, "dog".to_string()); - /// } - /// ``` - pub fn at(&'view self, index: usize) -> Result, DecoderError> { - if !self.is_list() { - return Err(DecoderError::RlpExpectedToBeList); - } - - // move to cached position if it's index is less or equal to - // current search index, otherwise move to beginning of list - let c = self.cache.get(); - let (mut bytes, to_skip) = match c.index <= index { - true => (try!(UntrustedRlp::consume(self.bytes, c.offset)), index - c.index), - false => (try!(self.consume_list_prefix()), index), - }; - - // skip up to x items - bytes = try!(UntrustedRlp::consume_items(bytes, to_skip)); - - // update the cache - self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); - - // construct new rlp - let found = try!(BasicDecoder::payload_info(bytes)); - Ok(UntrustedRlp::new(&bytes[0..found.header_len + found.value_len])) - } - - /// No value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![]; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.is_null()); - /// } - /// ``` - pub fn is_null(&self) -> bool { - self.bytes.len() == 0 - } - - /// Contains a zero-length string or zero-length list. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc0]; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.is_empty()); - /// } - /// ``` - pub fn is_empty(&self) -> bool { - !self.is_null() && (self.bytes[0] == 0xc0 || self.bytes[0] == 0x80) - } - - /// List value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.is_list()); - /// } - /// ``` - pub fn is_list(&self) -> bool { - !self.is_null() && self.bytes[0] >= 0xc0 - } - - /// String value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// assert!(rlp.at(1).unwrap().is_data()); - /// } - /// ``` - pub fn is_data(&self) -> bool { - !self.is_null() && self.bytes[0] < 0xc0 - } - - /// Int value - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc1, 0x10]; - /// let rlp = UntrustedRlp::new(&data); - /// assert_eq!(rlp.is_int(), false); - /// assert_eq!(rlp.at(0).unwrap().is_int(), true); - /// } - /// ``` - pub fn is_int(&self) -> bool { - if self.is_null() { - return false; - } - - match self.bytes[0] { - 0...0x80 => true, - 0x81...0xb7 => self.bytes[1] != 0, - b @ 0xb8...0xbf => self.bytes[1 + b as usize - 0xb7] != 0, - _ => false - } - } - - /// Get iterator over rlp-slices - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - /// let rlp = UntrustedRlp::new(&data); - /// let strings: Vec = rlp.iter() - /// .map(| i | String::decode_untrusted(&i)) - /// .map(| s | s.unwrap()) - /// .collect(); - /// } - /// ``` - pub fn iter(&'a self) -> UntrustedRlpIterator<'a> { - self.into_iter() - } - - /// consumes first found prefix - fn consume_list_prefix(&self) -> Result<&'a [u8], DecoderError> { - let item = try!(BasicDecoder::payload_info(self.bytes)); - let bytes = try!(UntrustedRlp::consume(self.bytes, item.header_len)); - Ok(bytes) - } - - /// consumes fixed number of items - fn consume_items(bytes: &'a [u8], items: usize) -> Result<&'a [u8], DecoderError> { - let mut result = bytes; - for _ in 0..items { - let i = try!(BasicDecoder::payload_info(result)); - result = try!(UntrustedRlp::consume(result, (i.header_len + i.value_len))); - } - Ok(result) - } - - - /// consumes slice prefix of length `len` - fn consume(bytes: &'a [u8], len: usize) -> Result<&'a [u8], DecoderError> { - match bytes.len() >= len { - true => Ok(&bytes[len..]), - false => Err(DecoderError::RlpIsTooShort), - } - } -} - -/// Iterator over rlp-slice list elements. -pub struct UntrustedRlpIterator<'a> { - rlp: &'a UntrustedRlp<'a>, - index: usize, -} - -impl<'a> IntoIterator for &'a UntrustedRlp<'a> { - type Item = UntrustedRlp<'a>; - type IntoIter = UntrustedRlpIterator<'a>; - - fn into_iter(self) -> Self::IntoIter { - UntrustedRlpIterator { - rlp: self, - index: 0, - } - } -} - -impl<'a> Iterator for UntrustedRlpIterator<'a> { - type Item = UntrustedRlp<'a>; - - fn next(&mut self) -> Option> { - let index = self.index; - let result = self.rlp.at(index).ok(); - self.index += 1; - result - } -} - -/// Iterator over trusted rlp-slice list elements. -pub struct RlpIterator<'a> { - rlp: &'a Rlp<'a>, - index: usize -} - -impl<'a> IntoIterator for &'a Rlp<'a> { - type Item = Rlp<'a>; - type IntoIter = RlpIterator<'a>; - - fn into_iter(self) -> Self::IntoIter { - RlpIterator { - rlp: self, - index: 0, - } - } -} - -impl<'a> Iterator for RlpIterator<'a> { - type Item = Rlp<'a>; - - fn next(&mut self) -> Option> { - let index = self.index; - let result = self.rlp.rlp.at(index).ok().map(| iter | { From::from(iter) }); - self.index += 1; - result - } -} - -/// Shortcut function to decode trusted rlp -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::rlp::*; -/// -/// fn main () { -/// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; -/// let animals: Vec = decode(&data); -/// assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); -/// } -/// ``` -pub fn decode(bytes: &[u8]) -> T where T: Decodable { - let rlp = Rlp::new(bytes); - T::decode(&rlp) -} - -pub trait Decodable: Sized { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result; - fn decode(rlp: &Rlp) -> Self { - Self::decode_untrusted(&rlp.rlp).unwrap() - } -} - -impl Decodable for T where T: FromBytes { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_data() { - true => BasicDecoder::read_value(rlp.bytes, | bytes | { - Ok(try!(T::from_bytes(bytes))) - }), - false => Err(DecoderError::RlpExpectedToBeData), - } - } -} - -impl Decodable for Vec where T: Decodable { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_list() { - true => rlp.iter().map(|rlp| T::decode_untrusted(&rlp)).collect(), - false => Err(DecoderError::RlpExpectedToBeList), - } - } -} - -impl Decodable for Vec { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - match rlp.is_data() { - true => BasicDecoder::read_value(rlp.bytes, | bytes | { - let mut res = vec![]; - res.extend(bytes); - Ok(res) - }), - false => Err(DecoderError::RlpExpectedToBeData), - } - } -} - -pub trait Decoder { - fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result; -} - -struct BasicDecoder; - -impl BasicDecoder { - /// Return first item info - fn payload_info(bytes: &[u8]) -> Result { - let item = match bytes.first().map(|&x| x) { - None => return Err(DecoderError::RlpIsTooShort), - Some(0...0x7f) => PayloadInfo::new(0, 1), - Some(l @ 0x80...0xb7) => PayloadInfo::new(1, l as usize - 0x80), - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - } - Some(l @ 0xc0...0xf7) => PayloadInfo::new(1, l as usize - 0xc0), - Some(l @ 0xf8...0xff) => { - let len_of_len = l as usize - 0xf7; - let header_len = 1 + len_of_len; - let value_len = try!(usize::from_bytes(&bytes[1..header_len])); - PayloadInfo::new(header_len, value_len) - }, - // we cant reach this place, but rust requires _ to be implemented - _ => { panic!(); } - }; - - match item.header_len + item.value_len <= bytes.len() { - true => Ok(item), - false => Err(DecoderError::RlpIsTooShort), - } - } -} - -impl Decoder for BasicDecoder { - fn read_value(bytes: &[u8], f: F) -> Result where F: FnOnce(&[u8]) -> Result { - match bytes.first().map(|&x| x) { - // rlp is too short - None => Err(DecoderError::RlpIsTooShort), - // single byt value - Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), - // 0-55 bytes - Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), - // longer than 55 bytes - Some(l @ 0xb8...0xbf) => { - let len_of_len = l as usize - 0xb7; - let begin_of_value = 1 as usize + len_of_len; - let len = try!(usize::from_bytes(&bytes[1..begin_of_value])); - Ok(try!(f(&bytes[begin_of_value..begin_of_value + len]))) - } - // we are reading value, not a list! - _ => { panic!(); } - } - } -} - -#[derive(Debug, Copy, Clone)] -struct ListInfo { - position: usize, - current: usize, - max: usize, -} - -impl ListInfo { - fn new(position: usize, max: usize) -> ListInfo { - ListInfo { - position: position, - current: 0, - max: max, - } - } -} - -/// Appendable rlp encoder. -pub struct RlpStream { - unfinished_lists: ElasticArray16, - encoder: BasicEncoder, -} - -impl RlpStream { - /// Initializes instance of empty `RlpStream`. - pub fn new() -> RlpStream { - RlpStream { - unfinished_lists: ElasticArray16::new(), - encoder: BasicEncoder::new(), - } - } - - /// Initializes the `RLPStream` as a list. - pub fn new_list(len: usize) -> RlpStream { - let mut stream = RlpStream::new(); - stream.append_list(len); - stream - } - - /// Apends value to the end of stream, chainable. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat").append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } - /// ``` - pub fn append<'a, E>(&'a mut self, object: &E) -> &'a mut RlpStream where E: Encodable { - // encode given value and add it at the end of the stream - object.encode(&mut self.encoder); - - // if list is finished, prepend the length - self.note_appended(1); - - // return chainable self - self - } - - /// Declare appending the list of given size, chainable. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_list(2).append(&"cat").append(&"dog"); - /// stream.append(&""); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); - /// } - /// ``` - pub fn append_list<'a>(&'a mut self, len: usize) -> &'a mut RlpStream { - match len { - 0 => { - // we may finish, if the appended list len is equal 0 - self.encoder.bytes.push(0xc0u8); - self.note_appended(1); - }, - _ => { - let position = self.encoder.bytes.len(); - self.unfinished_lists.push(ListInfo::new(position, len)); - }, - } - - // return chainable self - self - } - - /// Apends null to the end of stream, chainable. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append_empty_data().append_empty_data(); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc2, 0x80, 0x80]); - /// } - /// ``` - pub fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { - // self push raw item - self.encoder.bytes.push(0x80); - - // try to finish and prepend the length - self.note_appended(1); - - // return chainable self - self - } - - /// Appends raw (pre-serialised) RLP data. Use with caution. Chainable. - pub fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { - // push raw items - self.encoder.bytes.append_slice(bytes); - - // try to finish and prepend the length - self.note_appended(item_count); - - // return chainable self - self - } - - /// Clear the output stream so far. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(3); - /// stream.append(&"cat"); - /// stream.clear(); - /// stream.append(&"dog"); - /// let out = stream.out(); - /// assert_eq!(out, vec![0x83, b'd', b'o', b'g']); - /// } - pub fn clear(&mut self) { - // clear bytes - self.encoder.bytes.clear(); - - // clear lists - self.unfinished_lists.clear(); - } - - /// Returns true if stream doesnt expect any more items. - /// - /// ```rust - /// extern crate ethcore_util as util; - /// use util::rlp::*; - /// - /// fn main () { - /// let mut stream = RlpStream::new_list(2); - /// stream.append(&"cat"); - /// assert_eq!(stream.is_finished(), false); - /// stream.append(&"dog"); - /// assert_eq!(stream.is_finished(), true); - /// let out = stream.out(); - /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); - /// } - pub fn is_finished(&self) -> bool { - self.unfinished_lists.len() == 0 - } - - /// Streams out encoded bytes. - /// - /// panic! if stream is not finished. - pub fn out(self) -> Vec { - match self.is_finished() { - true => self.encoder.out().to_vec(), - false => panic!() - } - } - - /// Try to finish lists - fn note_appended(&mut self, inserted_items: usize) -> () { - if self.unfinished_lists.len() == 0 { - return; - } - - let back = self.unfinished_lists.len() - 1; - let should_finish = match self.unfinished_lists.get_mut(back) { - None => false, - Some(ref mut x) => { - x.current += inserted_items; - if x.current > x.max { - panic!("You cannot append more items then you expect!"); - } - x.current == x.max - } - }; - - if should_finish { - let x = self.unfinished_lists.pop().unwrap(); - let len = self.encoder.bytes.len() - x.position; - self.encoder.insert_list_len_at_pos(len, x.position); - self.note_appended(1); - } - } -} - -/// Shortcut function to encode structure into rlp. -/// -/// ```rust -/// extern crate ethcore_util as util; -/// use util::rlp::*; -/// -/// fn main () { -/// let animals = vec!["cat", "dog"]; -/// let out = encode(&animals); -/// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); -/// } -/// ``` -pub fn encode(object: &E) -> Vec where E: Encodable -{ - let mut encoder = BasicEncoder::new(); - object.encode(&mut encoder); - encoder.out().to_vec() -} - -pub trait Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder; -} - -pub trait Encoder { - fn emit_value(&mut self, bytes: &[u8]) -> (); - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); -} - -impl Encodable for T where T: ToBytes { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(&self.to_bytes()) - } -} - -impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_list(|e| { - // insert all list elements - for el in self.iter() { - el.encode(e); - } - }) - } -} - -impl Encodable for Vec where T: Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - let r: &[T] = self.as_ref(); - r.encode(encoder) - } -} - -/// lets treat bytes differently than other lists -/// they are a single value -impl<'a> Encodable for &'a [u8] { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(self) - } -} - -/// lets treat bytes differently than other lists -/// they are a single value -impl Encodable for Vec { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_value(self) - } -} - -struct BasicEncoder { - bytes: ElasticArray1024, -} - -impl BasicEncoder { - fn new() -> BasicEncoder { - BasicEncoder { bytes: ElasticArray1024::new() } - } - - /// inserts list prefix at given position - /// TODO: optimise it further? - fn insert_list_len_at_pos(&mut self, len: usize, pos: usize) -> () { - let mut res = vec![]; - match len { - 0...55 => res.push(0xc0u8 + len as u8), - _ => { - res.push(0xf7u8 + len.to_bytes_len() as u8); - res.extend(len.to_bytes()); - } - }; - - self.bytes.insert_slice(pos, &res); - } - - /// get encoded value - fn out(self) -> ElasticArray1024 { - self.bytes - } -} - -impl Encoder for BasicEncoder { - fn emit_value(&mut self, bytes: &[u8]) -> () { - match bytes.len() { - // just 0 - 0 => self.bytes.push(0x80u8), - // byte is its own encoding - 1 if bytes[0] < 0x80 => self.bytes.append_slice(bytes), - // (prefix + length), followed by the string - len @ 1 ... 55 => { - self.bytes.push(0x80u8 + len as u8); - self.bytes.append_slice(bytes); - } - // (prefix + length of length), followed by the length, followd by the string - len => { - self.bytes.push(0xb7 + len.to_bytes_len() as u8); - self.bytes.append_slice(&len.to_bytes()); - self.bytes.append_slice(bytes); - } - } - } - - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () - { - // get len before inserting a list - let before_len = self.bytes.len(); - - // insert all list elements - f(self); - - // get len after inserting a list - let after_len = self.bytes.len(); - - // diff is list len - let list_len = after_len - before_len; - self.insert_list_len_at_pos(list_len, before_len); - } -} - -#[cfg(test)] -mod tests { - extern crate json_tests; - use self::json_tests::execute_tests_from_directory; - use self::json_tests::rlp as rlptest; - use std::{fmt, cmp}; - use std::str::FromStr; - use rlp; - use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; - use uint::U256; - - #[test] - fn rlp_at() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - { - let rlp = UntrustedRlp::new(&data); - assert!(rlp.is_list()); - //let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); - let animals: Vec = rlp.as_val().unwrap(); - assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); - - let cat = rlp.at(0).unwrap(); - assert!(cat.is_data()); - assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); - //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); - assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); - - let dog = rlp.at(1).unwrap(); - assert!(dog.is_data()); - assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); - //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); - assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); - - let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_data()); - assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); - //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); - assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); - } - } - - #[test] - fn rlp_at_err() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; - { - let rlp = UntrustedRlp::new(&data); - assert!(rlp.is_list()); - - let cat_err = rlp.at(0).unwrap_err(); - assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); - - let dog_err = rlp.at(1).unwrap_err(); - assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); - } - } - - #[test] - fn rlp_iter() { - let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; - { - let rlp = UntrustedRlp::new(&data); - let mut iter = rlp.iter(); - - let cat = iter.next().unwrap(); - assert!(cat.is_data()); - assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); - - let dog = iter.next().unwrap(); - assert!(dog.is_data()); - assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); - - let none = iter.next(); - assert!(none.is_none()); - - let cat_again = rlp.at(0).unwrap(); - assert!(cat_again.is_data()); - assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); - } - } - - struct ETestPair(T, Vec) where T: rlp::Encodable; - - fn run_encode_tests(tests: Vec>) - where T: rlp::Encodable - { - for t in &tests { - let res = rlp::encode(&t.0); - assert_eq!(res, &t.1[..]); - } - } - - #[test] - fn encode_u16() { - let tests = vec![ - ETestPair(0u16, vec![0x80u8]), - ETestPair(0x100, vec![0x82, 0x01, 0x00]), - ETestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_u32() { - let tests = vec![ - ETestPair(0u32, vec![0x80u8]), - ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_u64() { - let tests = vec![ - ETestPair(0u64, vec![0x80u8]), - ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_u256() { - let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), - ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - ETestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_encode_tests(tests); - } - - #[test] - fn encode_str() { - let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), - ETestPair("dog", vec![0x83, b'd', b'o', b'g']), - ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), - ETestPair("", vec![0x80]), - ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; - run_encode_tests(tests); - } - - #[test] - fn encode_address() { - use hash::*; - - let tests = vec![ - ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), - vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) - ]; - run_encode_tests(tests); - } - - /// Vec (Bytes) is treated as a single value - #[test] - fn encode_vector_u8() { - let tests = vec![ - ETestPair(vec![], vec![0x80]), - ETestPair(vec![0u8], vec![0]), - ETestPair(vec![0x15], vec![0x15]), - ETestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_vector_u64() { - let tests = vec![ - ETestPair(vec![], vec![0xc0]), - ETestPair(vec![15u64], vec![0xc1, 0x0f]), - ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_encode_tests(tests); - } - - #[test] - fn encode_vector_str() { - let tests = vec![ETestPair(vec!["cat", "dog"], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_encode_tests(tests); - } - - #[test] - fn encode_vector_of_vectors_str() { - let tests = vec![ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_encode_tests(tests); - } - - struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; - - fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { - for t in &tests { - let res: T = rlp::decode(&t.1); - assert_eq!(res, t.0); - } - } - - /// Vec (Bytes) is treated as a single value - #[test] - fn decode_vector_u8() { - let tests = vec![ - DTestPair(vec![], vec![0x80]), - DTestPair(vec![0u8], vec![0]), - DTestPair(vec![0x15], vec![0x15]), - DTestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u16() { - let tests = vec![ - DTestPair(0u16, vec![0u8]), - DTestPair(0x100, vec![0x82, 0x01, 0x00]), - DTestPair(0xffff, vec![0x82, 0xff, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u32() { - let tests = vec![ - DTestPair(0u32, vec![0u8]), - DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), - DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u64() { - let tests = vec![ - DTestPair(0u64, vec![0u8]), - DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_u256() { - let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), - DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), - DTestPair(U256::from(0xffffffffu64), - vec![0x84, 0xff, 0xff, 0xff, 0xff]), - DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ - 000100000000000012f0") - .unwrap(), - vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, - 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_str() { - let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), - DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), - DTestPair("Marek".to_string(), - vec![0x85, b'M', b'a', b'r', b'e', b'k']), - DTestPair("".to_string(), vec![0x80]), - DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" - .to_string(), - vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', - b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', - b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', - b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', - b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', - b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', - b'e', b'l', b'i', b't'])]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_address() { - use hash::*; - - let tests = vec![ - DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), - vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, - 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, - 0xb3, 0x7d, 0x11, 0x06]) - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_vector_u64() { - let tests = vec![ - DTestPair(vec![], vec![0xc0]), - DTestPair(vec![15u64], vec![0xc1, 0x0f]), - DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), - DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), - ]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_vector_str() { - let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], - vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; - run_decode_tests(tests); - } - - #[test] - fn decode_untrusted_vector_of_vectors_str() { - let tests = vec![DTestPair(vec![vec!["cat".to_string()]], - vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; - run_decode_tests(tests); - } - - #[test] - fn test_rlp_json() { - println!("Json rlp test: "); - execute_tests_from_directory::("json-tests/json/rlp/stream/*.json", &mut | file, input, output | { - println!("file: {}", file); - - let mut stream = RlpStream::new(); - for operation in input.into_iter() { - match operation { - rlptest::Operation::Append(ref v) => stream.append(v), - rlptest::Operation::AppendList(len) => stream.append_list(len), - rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len), - rlptest::Operation::AppendEmpty => stream.append_empty_data() - }; - } - - assert_eq!(stream.out(), output); - }); - } - -} diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index 2effb7297..ce43c10a5 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -1,5 +1,4 @@ -use super::faces::{View, Decodable, DecoderError}; -use super::untrusted_rlp::*; +use rlp::{View, Decodable, DecoderError, UntrustedRlp, PayloadInfo, Prototype}; impl<'a> From> for Rlp<'a> { fn from(rlp: UntrustedRlp<'a>) -> Rlp<'a> { diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 379852083..557c4af25 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -1,6 +1,6 @@ use elastic_array::*; use bytes::ToBytes; -use super::faces::{Stream, Encoder, Encodable}; +use rlp::{Stream, Encoder, Encodable}; #[derive(Debug, Copy, Clone)] struct ListInfo { diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs new file mode 100644 index 000000000..e44953e07 --- /dev/null +++ b/src/rlp/tests.rs @@ -0,0 +1,345 @@ +extern crate json_tests; +use self::json_tests::execute_tests_from_directory; +use self::json_tests::rlp as rlptest; +use std::{fmt, cmp}; +use std::str::FromStr; +use rlp; +use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; +use uint::U256; + +#[test] +fn rlp_at() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = UntrustedRlp::new(&data); + assert!(rlp.is_list()); + //let animals = as rlp::Decodable>::decode_untrusted(&rlp).unwrap(); + let animals: Vec = rlp.as_val().unwrap(); + assert_eq!(animals, vec!["cat".to_string(), "dog".to_string()]); + + let cat = rlp.at(0).unwrap(); + assert!(cat.is_data()); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); + assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); + + let dog = rlp.at(1).unwrap(); + assert!(dog.is_data()); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); + assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); + + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_data()); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); + assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); + } +} + +#[test] +fn rlp_at_err() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o']; + { + let rlp = UntrustedRlp::new(&data); + assert!(rlp.is_list()); + + let cat_err = rlp.at(0).unwrap_err(); + assert_eq!(cat_err, rlp::DecoderError::RlpIsTooShort); + + let dog_err = rlp.at(1).unwrap_err(); + assert_eq!(dog_err, rlp::DecoderError::RlpIsTooShort); + } +} + +#[test] +fn rlp_iter() { + let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; + { + let rlp = UntrustedRlp::new(&data); + let mut iter = rlp.iter(); + + let cat = iter.next().unwrap(); + assert!(cat.is_data()); + assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + + let dog = iter.next().unwrap(); + assert!(dog.is_data()); + assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + + let none = iter.next(); + assert!(none.is_none()); + + let cat_again = rlp.at(0).unwrap(); + assert!(cat_again.is_data()); + assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + } +} + +struct ETestPair(T, Vec) where T: rlp::Encodable; + +fn run_encode_tests(tests: Vec>) + where T: rlp::Encodable +{ + for t in &tests { + let res = rlp::encode(&t.0); + assert_eq!(res, &t.1[..]); + } +} + +#[test] +fn encode_u16() { + let tests = vec![ + ETestPair(0u16, vec![0x80u8]), + ETestPair(0x100, vec![0x82, 0x01, 0x00]), + ETestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_u32() { + let tests = vec![ + ETestPair(0u32, vec![0x80u8]), + ETestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + ETestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_u64() { + let tests = vec![ + ETestPair(0u64, vec![0x80u8]), + ETestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_u256() { + let tests = vec![ETestPair(U256::from(0u64), vec![0x80u8]), + ETestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + ETestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ETestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_encode_tests(tests); +} + +#[test] +fn encode_str() { + let tests = vec![ETestPair("cat", vec![0x83, b'c', b'a', b't']), + ETestPair("dog", vec![0x83, b'd', b'o', b'g']), + ETestPair("Marek", vec![0x85, b'M', b'a', b'r', b'e', b'k']), + ETestPair("", vec![0x80]), + ETestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit", + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_encode_tests(tests); +} + +#[test] +fn encode_address() { + use hash::*; + + let tests = vec![ + ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) + ]; + run_encode_tests(tests); +} + +/// Vec (Bytes) is treated as a single value +#[test] +fn encode_vector_u8() { + let tests = vec![ + ETestPair(vec![], vec![0x80]), + ETestPair(vec![0u8], vec![0]), + ETestPair(vec![0x15], vec![0x15]), + ETestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_vector_u64() { + let tests = vec![ + ETestPair(vec![], vec![0xc0]), + ETestPair(vec![15u64], vec![0xc1, 0x0f]), + ETestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + ETestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_encode_tests(tests); +} + +#[test] +fn encode_vector_str() { + let tests = vec![ETestPair(vec!["cat", "dog"], + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_encode_tests(tests); +} + +#[test] +fn encode_vector_of_vectors_str() { + let tests = vec![ETestPair(vec![vec!["cat"]], vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_encode_tests(tests); +} + +struct DTestPair(T, Vec) where T: rlp::Decodable + fmt::Debug + cmp::Eq; + +fn run_decode_tests(tests: Vec>) where T: rlp::Decodable + fmt::Debug + cmp::Eq { + for t in &tests { + let res: T = rlp::decode(&t.1); + assert_eq!(res, t.0); + } +} + +/// Vec (Bytes) is treated as a single value +#[test] +fn decode_vector_u8() { + let tests = vec![ + DTestPair(vec![], vec![0x80]), + DTestPair(vec![0u8], vec![0]), + DTestPair(vec![0x15], vec![0x15]), + DTestPair(vec![0x40, 0x00], vec![0x82, 0x40, 0x00]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u16() { + let tests = vec![ + DTestPair(0u16, vec![0u8]), + DTestPair(0x100, vec![0x82, 0x01, 0x00]), + DTestPair(0xffff, vec![0x82, 0xff, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u32() { + let tests = vec![ + DTestPair(0u32, vec![0u8]), + DTestPair(0x10000, vec![0x83, 0x01, 0x00, 0x00]), + DTestPair(0xffffff, vec![0x83, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u64() { + let tests = vec![ + DTestPair(0u64, vec![0u8]), + DTestPair(0x1000000, vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(0xFFFFFFFF, vec![0x84, 0xff, 0xff, 0xff, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_u256() { + let tests = vec![DTestPair(U256::from(0u64), vec![0x80u8]), + DTestPair(U256::from(0x1000000u64), vec![0x84, 0x01, 0x00, 0x00, 0x00]), + DTestPair(U256::from(0xffffffffu64), + vec![0x84, 0xff, 0xff, 0xff, 0xff]), + DTestPair(U256::from_str("8090a0b0c0d0e0f00910203040506077000000000000\ + 000100000000000012f0") + .unwrap(), + vec![0xa0, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0, + 0x09, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0])]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_str() { + let tests = vec![DTestPair("cat".to_string(), vec![0x83, b'c', b'a', b't']), + DTestPair("dog".to_string(), vec![0x83, b'd', b'o', b'g']), + DTestPair("Marek".to_string(), + vec![0x85, b'M', b'a', b'r', b'e', b'k']), + DTestPair("".to_string(), vec![0x80]), + DTestPair("Lorem ipsum dolor sit amet, consectetur adipisicing elit" + .to_string(), + vec![0xb8, 0x38, b'L', b'o', b'r', b'e', b'm', b' ', b'i', + b'p', b's', b'u', b'm', b' ', b'd', b'o', b'l', b'o', + b'r', b' ', b's', b'i', b't', b' ', b'a', b'm', b'e', + b't', b',', b' ', b'c', b'o', b'n', b's', b'e', b'c', + b't', b'e', b't', b'u', b'r', b' ', b'a', b'd', b'i', + b'p', b'i', b's', b'i', b'c', b'i', b'n', b'g', b' ', + b'e', b'l', b'i', b't'])]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_address() { + use hash::*; + + let tests = vec![ + DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, + 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, + 0xb3, 0x7d, 0x11, 0x06]) + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_vector_u64() { + let tests = vec![ + DTestPair(vec![], vec![0xc0]), + DTestPair(vec![15u64], vec![0xc1, 0x0f]), + DTestPair(vec![1, 2, 3, 7, 0xff], vec![0xc6, 1, 2, 3, 7, 0x81, 0xff]), + DTestPair(vec![0xffffffff, 1, 2, 3, 7, 0xff], vec![0xcb, 0x84, 0xff, 0xff, 0xff, 0xff, 1, 2, 3, 7, 0x81, 0xff]), + ]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_vector_str() { + let tests = vec![DTestPair(vec!["cat".to_string(), "dog".to_string()], + vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g'])]; + run_decode_tests(tests); +} + +#[test] +fn decode_untrusted_vector_of_vectors_str() { + let tests = vec![DTestPair(vec![vec!["cat".to_string()]], + vec![0xc5, 0xc4, 0x83, b'c', b'a', b't'])]; + run_decode_tests(tests); +} + +#[test] +fn test_rlp_json() { + println!("Json rlp test: "); + execute_tests_from_directory::("json-tests/json/rlp/stream/*.json", &mut | file, input, output | { + println!("file: {}", file); + + let mut stream = RlpStream::new(); + for operation in input.into_iter() { + match operation { + rlptest::Operation::Append(ref v) => stream.append(v), + rlptest::Operation::AppendList(len) => stream.append_list(len), + rlptest::Operation::AppendRaw(ref raw, len) => stream.append_raw(raw, len), + rlptest::Operation::AppendEmpty => stream.append_empty_data() + }; + } + + assert_eq!(stream.out(), output); + }); +} + diff --git a/src/rlp/faces.rs b/src/rlp/traits.rs similarity index 92% rename from src/rlp/faces.rs rename to src/rlp/traits.rs index 078620de2..966078b08 100644 --- a/src/rlp/faces.rs +++ b/src/rlp/traits.rs @@ -1,32 +1,4 @@ -use std::fmt; -use std::error::Error as StdError; -use bytes::FromBytesError; - -#[derive(Debug, PartialEq, Eq)] -pub enum DecoderError { - FromBytesError(FromBytesError), - RlpIsTooShort, - RlpExpectedToBeList, - RlpExpectedToBeData, -} - -impl StdError for DecoderError { - fn description(&self) -> &str { - "builder error" - } -} - -impl fmt::Display for DecoderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(&self, f) - } -} - -impl From for DecoderError { - fn from(err: FromBytesError) -> DecoderError { - DecoderError::FromBytesError(err) - } -} +use rlp::DecoderError; pub trait Decoder { fn read_value(&self, f: F) -> Result diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 9e2e2f522..19dd5e99c 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,6 +1,6 @@ use std::cell::Cell; use bytes::{FromBytes}; -use super::faces::{View, Decoder, Decodable, DecoderError}; +use rlp::{View, Decoder, Decodable, DecoderError}; /// rlp offset #[derive(Copy, Clone, Debug)] From d5ab429e777b86d141daf61ec280d36cd39973f5 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 13:02:32 +0100 Subject: [PATCH 240/379] changed order of reexports --- src/rlp/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 49043d65f..cb36426c9 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -41,8 +41,8 @@ mod tests; pub use self::errors::DecoderError; pub use self::traits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; +pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; pub use self::rlp::{Rlp, RlpIterator}; -pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, Prototype, PayloadInfo}; pub use self::rlpstream::{RlpStream}; /// Shortcut function to decode trusted rlp From 1c58ba47642c0cf0397380e83275625e34000213 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 13:22:24 +0100 Subject: [PATCH 241/379] . --- benches/rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 62f6c6112..1160e311f 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -91,6 +91,6 @@ fn bench_stream_1000_empty_lists(b: &mut Bencher) { for _ in 0..1000 { stream.append_list(0); } - //let _ = stream.out(); + let _ = stream.out(); }); } From b657a15100ef2bc0f93b6e15cc6fda55e1338e28 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 14:44:22 +0100 Subject: [PATCH 242/379] val_at rlp method --- src/rlp/rlp.rs | 12 ++++++++++-- src/rlp/traits.rs | 2 ++ src/rlp/untrusted_rlp.rs | 4 ++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index ce43c10a5..ac830cc9c 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -84,16 +84,24 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { fn as_val(&self) -> Result where T: Decodable { self.rlp.as_val() } + + fn val_at(&self, index: usize) -> Result where T: Decodable { + self.at(index).rlp.as_val() + } } impl <'a, 'view> Rlp<'a> where 'a: 'view { - fn reader_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { + fn view_as_val(r: &R) -> T where R: View<'a, 'view>, T: Decodable { let res: Result = r.as_val(); res.unwrap_or_else(|_| panic!()) } pub fn as_val(&self) -> T where T: Decodable { - Self::reader_as_val(self) + Self::view_as_val(self) + } + + pub fn val_at(&self, index: usize) -> T where T: Decodable { + Self::view_as_val(&self.at(index)) } } diff --git a/src/rlp/traits.rs b/src/rlp/traits.rs index 966078b08..1127ca3db 100644 --- a/src/rlp/traits.rs +++ b/src/rlp/traits.rs @@ -179,6 +179,8 @@ pub trait View<'a, 'view>: Sized { fn iter(&'view self) -> Self::Iter; fn as_val(&self) -> Result where T: Decodable; + + fn val_at(&self, index: usize) -> Result where T: Decodable; } pub trait Encoder { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 19dd5e99c..eae253eca 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -176,6 +176,10 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { // optimize, so it doesn't use clone (although This clone is cheap) T::decode(&BasicDecoder::new(self.clone())) } + + fn val_at(&self, index: usize) -> Result where T: Decodable { + self.at(index).unwrap().as_val() + } } impl<'a> UntrustedRlp<'a> { From 0e343d078cc9a64afc8b6ea65fe2f0f9dbd0c6f1 Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 8 Dec 2015 19:01:37 +0100 Subject: [PATCH 243/379] rlp encodable and decodable for options --- src/rlp/mod.rs | 3 +-- src/rlp/rlpstream.rs | 19 ++++++++++++++----- src/rlp/untrusted_rlp.rs | 12 ++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index cb36426c9..981e5f973 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -74,8 +74,7 @@ pub fn decode(bytes: &[u8]) -> T where T: Decodable { /// assert_eq!(out, vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']); /// } /// ``` -pub fn encode(object: &E) -> Vec where E: Encodable -{ +pub fn encode(object: &E) -> Vec where E: Encodable { let mut stream = RlpStream::new(); stream.append(object); stream.out() diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 557c4af25..301a39053 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -207,13 +207,13 @@ impl Encoder for BasicEncoder { } impl Encodable for T where T: ToBytes { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(&self.to_bytes()) } } impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_list(|e| { // insert all list elements for el in self.iter() { @@ -224,7 +224,7 @@ impl<'a, T> Encodable for &'a [T] where T: Encodable + 'a { } impl Encodable for Vec where T: Encodable { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { let r: &[T] = self.as_ref(); r.encode(encoder) } @@ -233,7 +233,7 @@ impl Encodable for Vec where T: Encodable { /// lets treat bytes differently than other lists /// they are a single value impl<'a> Encodable for &'a [u8] { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(self) } } @@ -241,7 +241,16 @@ impl<'a> Encodable for &'a [u8] { /// lets treat bytes differently than other lists /// they are a single value impl Encodable for Vec { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { + fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(self) } } + +impl Encodable for Option where T: Encodable { + fn encode(&self, encoder: &mut E) where E: Encoder { + match *self { + Some(ref x) => x.encode(encoder), + None => encoder.emit_value(&[]) + } + } +} diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index eae253eca..ec68fce99 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -340,3 +340,15 @@ impl Decodable for Vec { }) } } + +impl Decodable for Option where T: Decodable { + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_value(| bytes | { + let res = match bytes.len() { + 0 => None, + _ => Some(try!(T::decode(decoder))) + }; + Ok(res) + }) + } +} From f10e72caeb3f8fac76610f1e697184a06a891341 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Dec 2015 02:29:34 +0100 Subject: [PATCH 244/379] fixed rlp benchmarks, added triehash benchmarks --- benches/rlp.rs | 14 +++++----- benches/trie.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 11 deletions(-) diff --git a/benches/rlp.rs b/benches/rlp.rs index 1160e311f..8bc7599e4 100644 --- a/benches/rlp.rs +++ b/benches/rlp.rs @@ -11,7 +11,7 @@ extern crate ethcore_util; use test::Bencher; use std::str::FromStr; -use ethcore_util::rlp::{RlpStream, Rlp, Decodable}; +use ethcore_util::rlp::*; use ethcore_util::uint::U256; #[bench] @@ -30,7 +30,7 @@ fn bench_decode_u64_value(b: &mut Bencher) { // u64 let data = vec![0x88, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; let rlp = Rlp::new(&data); - let _ = u64::decode(&rlp); + let _: u64 = rlp.as_val(); }); } @@ -54,7 +54,7 @@ fn bench_decode_u256_value(b: &mut Bencher) { 0x30, 0x40, 0x50, 0x60, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0xf0]; let rlp = Rlp::new(&data); - let _ = U256::decode(&rlp); + let _ : U256 = rlp.as_val(); }); } @@ -76,11 +76,11 @@ fn bench_decode_nested_empty_lists(b: &mut Bencher) { // [ [], [[]], [ [], [[]] ] ] let data = vec![0xc7, 0xc0, 0xc1, 0xc0, 0xc3, 0xc0, 0xc1, 0xc0]; let rlp = Rlp::new(&data); - let _v0: Vec = Decodable::decode(&rlp.at(0)); - let _v1: Vec> = Decodable::decode(&rlp.at(1)); + let _v0: Vec = rlp.val_at(0); + let _v1: Vec> = rlp.val_at(1); let nested_rlp = rlp.at(2); - let _v2a: Vec = Decodable::decode(&nested_rlp.at(0)); - let _v2b: Vec> = Decodable::decode(&nested_rlp.at(1)); + let _v2a: Vec = nested_rlp.val_at(0); + let _v2b: Vec> = nested_rlp.val_at(1); }); } diff --git a/benches/trie.rs b/benches/trie.rs index 2865c5778..a8cd407cf 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -10,6 +10,7 @@ use test::Bencher; use ethcore_util::hash::*; use ethcore_util::bytes::*; use ethcore_util::trie::*; +use ethcore_util::triehash::*; use ethcore_util::sha3::*; @@ -40,7 +41,7 @@ fn random_value(seed: &mut H256) -> Bytes { } #[bench] -fn insertions_six_high(b: &mut Bencher) { +fn trie_insertions_six_high(b: &mut Bencher) { let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); for _ in 0..1000 { @@ -58,7 +59,22 @@ fn insertions_six_high(b: &mut Bencher) { } #[bench] -fn insertions_six_mid(b: &mut Bencher) { +fn triehash_insertions_six_high(b: &mut Bencher) { + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_bytes(6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(&||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn trie_insertions_six_mid(b: &mut Bencher) { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); @@ -77,7 +93,22 @@ fn insertions_six_mid(b: &mut Bencher) { } #[bench] -fn insertions_random_mid(b: &mut Bencher) { +fn triehash_insertions_six_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + b.iter(||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn trie_insertions_random_mid(b: &mut Bencher) { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); @@ -96,7 +127,23 @@ fn insertions_random_mid(b: &mut Bencher) { } #[bench] -fn insertions_six_low(b: &mut Bencher) { +fn triehash_insertions_random_mid(b: &mut Bencher) { + let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 1, 5, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + trie_root(d.clone()); + }) +} + +#[bench] +fn trie_insertions_six_low(b: &mut Bencher) { let alphabet = b"abcdef"; let mut d: Vec<(Bytes, Bytes)> = Vec::new(); let mut seed = H256::new(); @@ -114,6 +161,22 @@ fn insertions_six_low(b: &mut Bencher) { }) } +#[bench] +fn triehash_insertions_six_low(b: &mut Bencher) { + let alphabet = b"abcdef"; + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..1000 { + let k = random_word(alphabet, 6, 0, &mut seed); + let v = random_value(&mut seed); + d.push((k, v)) + } + + b.iter(||{ + trie_root(d.clone()); + }) +} + #[bench] fn sha3x1000(b: &mut Bencher) { b.iter(||{ From d2b1b92f9eca2a5721e43e2abfa2d3bc931937b8 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Dec 2015 11:05:27 +0100 Subject: [PATCH 245/379] removed unused macros.rs --- src/lib.rs | 1 - src/macros.rs | 11 ----------- 2 files changed, 12 deletions(-) delete mode 100644 src/macros.rs diff --git a/src/lib.rs b/src/lib.rs index 883cac2f2..12998529a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,7 +44,6 @@ extern crate secp256k1; extern crate arrayvec; extern crate elastic_array; -pub mod macros; pub mod error; pub mod hash; pub mod uint; diff --git a/src/macros.rs b/src/macros.rs deleted file mode 100644 index 69286a340..000000000 --- a/src/macros.rs +++ /dev/null @@ -1,11 +0,0 @@ -macro_rules! map( - { $($key:expr => $value:expr),+ } => { - { - let mut m = ::std::collections::HashMap::new(); - $( - m.insert($key, $value); - )+ - m - } - }; -); From 6db12408dcf9ac01c9d91dea9b02136f97a87a50 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 9 Dec 2015 14:50:01 +0100 Subject: [PATCH 246/379] fixed uint multiplication --- src/uint.rs | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index e017de385..234867ea9 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -82,6 +82,15 @@ macro_rules! construct_uint { bytes[i] = (arr[pos] >> ((rev % 8) * 8)) as u8; } } + + #[inline] + pub fn exp10(n: usize) -> $name { + match n { + 0 => $name::from(1u64), + _ => $name::exp10(n - 1) * $name::from(10u64) + } + } + /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -167,12 +176,12 @@ macro_rules! construct_uint { type Output = $name; fn mul(self, other: $name) -> $name { - let mut me = self; + let mut res = $name::from(0u64); // TODO: be more efficient about this for i in 0..(2 * $n_words) { - me = (me + me.mul_u32((other >> (32 * i)).low_u32())) << (32 * i); + res = res + (self.mul_u32((other >> (32 * i)).low_u32()) << (32 * i)); } - me + res } } @@ -516,5 +525,37 @@ mod tests { assert_eq!(add >> 64, U256([0xDEADBEEFDEADBEEF, 0, 0, 0])); assert_eq!(add << 64, U256([0, 0xDEADBEEFDEADBEEF, 0xDEADBEEFDEADBEEF, 0])); } + + #[test] + pub fn uint256_exp10() { + assert_eq!(U256::exp10(0), U256::from(1u64)); + println!("\none: {:?}", U256::from(1u64)); + println!("ten: {:?}", U256::from(10u64)); + assert_eq!(U256::from(2u64) * U256::from(10u64), U256::from(20u64)); + assert_eq!(U256::exp10(1), U256::from(10u64)); + assert_eq!(U256::exp10(2), U256::from(100u64)); + assert_eq!(U256::exp10(5), U256::from(100000u64)); + } + + #[test] + pub fn uint256_mul32() { + assert_eq!(U256::from(0u64).mul_u32(2), U256::from(0u64)); + assert_eq!(U256::from(1u64).mul_u32(2), U256::from(2u64)); + assert_eq!(U256::from(10u64).mul_u32(2), U256::from(20u64)); + assert_eq!(U256::from(10u64).mul_u32(5), U256::from(50u64)); + assert_eq!(U256::from(1000u64).mul_u32(50), U256::from(50000u64)); + } + + #[test] + pub fn uint256_mul() { + assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); + } + + #[test] + fn uint256_div() { + assert_eq!(U256::from(10u64) / U256::from(1u64), U256::from(10u64)); + assert_eq!(U256::from(10u64) / U256::from(2u64), U256::from(5u64)); + assert_eq!(U256::from(10u64) / U256::from(3u64), U256::from(3u64)); + } } From 1a404167249bcc7e1db801f40d0db4843e3341ff Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Dec 2015 01:41:44 +0100 Subject: [PATCH 247/379] Makefile for cross-compilation --- Makefile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..ad3c5230f --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +# Makefile for cross-compilation +IOS_ARCHS = i386-apple-ios x86_64-apple-ios armv7-apple-ios armv7s-apple-ios aarch64-apple-ios +IOS_LIB = libethcore_util.a + +ios: $(IOS_LIB) + +.PHONY: $(IOS_ARCHS) +$(IOS_ARCHS): %: + multirust run ios cargo build --target $@ + +$(IOS_LIB): $(IOS_ARCHS) + lipo -create -output $@ $(foreach arch,$(IOS_ARCHS),$(wildcard target/$(arch)/debug/$(IOS_LIB))) From 4210b957ac2ab0558271651dd83dcb3d2db14fff Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Dec 2015 15:48:25 +0100 Subject: [PATCH 248/379] triedb db is borrowed instead of being owned --- src/trie.rs | 110 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 3c7e8bfb2..745dc9c48 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -2,7 +2,6 @@ extern crate rand; use std::fmt; -use memorydb::*; use sha3::*; use hashdb::*; use hash::*; @@ -279,10 +278,14 @@ impl <'a>Node<'a> { /// /// # Example /// ``` -/// extern crate ethcore_util; -/// use ethcore_util::trie::*; +/// extern crate ethcore_util as util; +/// use util::trie::*; +/// use util::hashdb::*; +/// use util::memorydb::*; +/// /// fn main() { -/// let mut t = TrieDB::new_memory(); +/// let mut memdb = MemoryDB::new(); +/// let mut t = TrieDB::new(&mut memdb); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), SHA3_NULL_RLP); /// t.insert(b"foo", b"bar"); @@ -294,8 +297,8 @@ impl <'a>Node<'a> { /// assert!(t.db_items_remaining().is_empty()); /// } /// ``` -pub struct TrieDB { - db: Box, +pub struct TrieDB<'db, T> where T: 'db + HashDB { + db: &'db mut T, root: H256, pub hash_count: usize, } @@ -306,18 +309,23 @@ enum MaybeChanged<'a> { Changed(Bytes), } -impl TrieDB { - /// Create a new trie with the boxed backing database `box_db`. - pub fn new_boxed(db_box: Box) -> Self { let mut r = TrieDB{ db: db_box, root: H256::new(), hash_count: 0 }; r.root = r.db.insert(&NULL_RLP); r } +impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { + /// Create a new trie with the backing database `db`. + pub fn new(db: &'db mut T) -> Self { + let mut r = TrieDB{ + db: db, + root: H256::new(), + hash_count: 0 + }; - /// Convenience function to create a new trie with the backing database `db`. - pub fn new(db: T) -> Self where T: HashDB + 'static { Self::new_boxed(Box::new(db)) } - - /// Convenience function to create a new trie with a new `MemoryDB` based backing database. - pub fn new_memory() -> Self { Self::new(MemoryDB::new()) } + r.root = r.db.insert(&NULL_RLP); + r + } /// Get the backing database. - pub fn db(&self) -> &HashDB { self.db.as_ref() } + pub fn db(&'db self) -> &'db T { + self.db + } /// Determine all the keys in the backing database that belong to the trie. pub fn keys(&self) -> Vec { @@ -868,7 +876,7 @@ impl TrieDB { } } -impl Trie for TrieDB { +impl<'db, T> Trie for TrieDB<'db, T> where T: 'db + HashDB { fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { @@ -891,7 +899,7 @@ impl Trie for TrieDB { } } -impl fmt::Debug for TrieDB { +impl<'db, T> fmt::Debug for TrieDB<'db, T> where T: 'db + HashDB { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); @@ -906,6 +914,8 @@ mod tests { use self::json_tests::{trie, execute_tests_from_directory}; use triehash::*; use hash::*; + use hashdb::*; + use memorydb::*; use super::*; use nibbleslice::*; use rlp; @@ -934,8 +944,8 @@ mod tests { } } - fn populate_trie(v: &Vec<(Vec, Vec)>) -> TrieDB { - let mut t = TrieDB::new_memory(); + fn populate_trie<'db, T>(db: &'db mut T, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { + let mut t = TrieDB::new(db); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -944,7 +954,7 @@ mod tests { t } - fn unpopulate_trie(t: &mut TrieDB, v: &Vec<(Vec, Vec)>) { + fn unpopulate_trie<'a, 'db, T>(t: &mut TrieDB<'db, T>, v: &Vec<(Vec, Vec)>) where T: 'db + HashDB { for i in v.iter() { let key: &[u8]= &i.0; t.remove(&key); @@ -994,7 +1004,8 @@ mod tests { } let real = trie_root(x.clone()); - let mut memtrie = populate_trie(&x); + let mut memdb = MemoryDB::new(); + let mut memtrie = populate_trie(&mut memdb, &x); if *memtrie.root() != real || !memtrie.db_items_remaining().is_empty() { println!("TRIE MISMATCH"); println!(""); @@ -1023,14 +1034,16 @@ mod tests { #[test] fn init() { - let t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let t = TrieDB::new(&mut memdb); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } #[test] fn insert_on_empty() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1039,12 +1052,14 @@ mod tests { fn remove_to_empty() { let big_value = b"00000000000000000000000000000000"; - let mut t1 = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t1 = TrieDB::new(&mut memdb); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); trace!("keys remaining {:?}", t1.db_items_remaining()); assert!(t1.db_items_remaining().is_empty()); - let mut t2 = TrieDB::new_memory(); + let mut memdb2 = MemoryDB::new(); + let mut t2 = TrieDB::new(&mut memdb2); t2.insert(&[0x01], &big_value.to_vec()); t2.insert(&[0x01, 0x23], &big_value.to_vec()); t2.insert(&[0x01, 0x34], &big_value.to_vec()); @@ -1058,7 +1073,8 @@ mod tests { #[test] fn insert_replace_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1066,7 +1082,8 @@ mod tests { #[test] fn insert_make_branch_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ @@ -1077,7 +1094,8 @@ mod tests { #[test] fn insert_into_branch_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1090,7 +1108,8 @@ mod tests { #[test] fn insert_value_into_branch_root() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[], &[0x0]); assert_eq!(*t.root(), trie_root(vec![ @@ -1101,7 +1120,8 @@ mod tests { #[test] fn insert_split_leaf() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); assert_eq!(*t.root(), trie_root(vec![ @@ -1112,7 +1132,8 @@ mod tests { #[test] fn insert_split_extenstion() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01, 0x23, 0x45], &[0x01]); t.insert(&[0x01, 0xf3, 0x45], &[0x02]); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); @@ -1128,7 +1149,8 @@ mod tests { let big_value0 = b"00000000000000000000000000000000"; let big_value1 = b"11111111111111111111111111111111"; - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], big_value0); t.insert(&[0x11u8, 0x23], big_value1); assert_eq!(*t.root(), trie_root(vec![ @@ -1141,7 +1163,8 @@ mod tests { fn insert_duplicate_value() { let big_value = b"00000000000000000000000000000000"; - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], big_value); t.insert(&[0x11u8, 0x23], big_value); assert_eq!(*t.root(), trie_root(vec![ @@ -1199,20 +1222,23 @@ mod tests { #[test] fn test_at_empty() { - let t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let t = TrieDB::new(&mut memdb); assert_eq!(t.at(&[0x5]), None); } #[test] fn test_at_one() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); } #[test] fn test_at_three() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1224,7 +1250,8 @@ mod tests { #[test] fn test_print_trie() { - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); @@ -1244,10 +1271,12 @@ mod tests { x.push((key, rlp::encode(&j))); } let real = trie_root(x.clone()); - let memtrie = populate_trie(&x); + let mut memdb = MemoryDB::new(); + let memtrie = populate_trie(&mut memdb, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); - let memtrie_sorted = populate_trie(&y); + let mut memdb2 = MemoryDB::new(); + let memtrie_sorted = populate_trie(&mut memdb2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1273,7 +1302,8 @@ mod tests { execute_tests_from_directory::("json-tests/json/trie/*.json", &mut | file, input, output | { println!("file: {}", file); - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for operation in input.into_iter() { match operation { trie::Operation::Insert(key, value) => t.insert(&key, &value), From 8c9fe2fbe5725275953195a51c2242309906c11c Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 10 Dec 2015 15:57:47 +0100 Subject: [PATCH 249/379] fixed benches --- benches/trie.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index a8cd407cf..c12a8f752 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -10,6 +10,8 @@ use test::Bencher; use ethcore_util::hash::*; use ethcore_util::bytes::*; use ethcore_util::trie::*; +use ethcore_util::hashdb::*; +use ethcore_util::memorydb::*; use ethcore_util::triehash::*; use ethcore_util::sha3::*; @@ -51,7 +53,8 @@ fn trie_insertions_six_high(b: &mut Bencher) { } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -84,7 +87,8 @@ fn trie_insertions_six_mid(b: &mut Bencher) { d.push((k, v)) } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -119,7 +123,8 @@ fn trie_insertions_random_mid(b: &mut Bencher) { } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -154,7 +159,8 @@ fn trie_insertions_six_low(b: &mut Bencher) { } b.iter(||{ - let mut t = TrieDB::new_memory(); + let mut memdb = MemoryDB::new(); + let mut t = TrieDB::new(&mut memdb); for i in d.iter() { t.insert(&i.0, &i.1); } From b487f8f20e6c8a5a4ef6b125576ca618edd076f6 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 03:00:39 +0100 Subject: [PATCH 250/379] TrieDB root is borrowed --- src/trie.rs | 108 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 79 insertions(+), 29 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 745dc9c48..2909e1314 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -282,10 +282,12 @@ impl <'a>Node<'a> { /// use util::trie::*; /// use util::hashdb::*; /// use util::memorydb::*; +/// use util::hash::*; /// /// fn main() { /// let mut memdb = MemoryDB::new(); -/// let mut t = TrieDB::new(&mut memdb); +/// let mut root = H256::new(); +/// let mut t = TrieDB::new(&mut memdb, &mut root); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), SHA3_NULL_RLP); /// t.insert(b"foo", b"bar"); @@ -299,7 +301,7 @@ impl <'a>Node<'a> { /// ``` pub struct TrieDB<'db, T> where T: 'db + HashDB { db: &'db mut T, - root: H256, + root: &'db mut H256, pub hash_count: usize, } @@ -310,18 +312,32 @@ enum MaybeChanged<'a> { } impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { - /// Create a new trie with the backing database `db`. - pub fn new(db: &'db mut T) -> Self { + /// Create a new trie with the backing database `db` and empty `root` + /// Initialise to the state entailed by the genesis block. + /// This guarantees the trie is built correctly. + pub fn new(db: &'db mut T, root: &'db mut H256) -> Self { let mut r = TrieDB{ db: db, - root: H256::new(), + root: root, hash_count: 0 }; - r.root = r.db.insert(&NULL_RLP); + // set root rlp + *r.root = r.db.insert(&NULL_RLP); r } + /// Create a new trie with the backing database `db` and `root` + /// Panics, if `root` does not exist + pub fn new_pre_existing(db: &'db mut T, root: &'db mut H256) -> Self { + assert!(db.exists(root)); + TrieDB { + db: db, + root: root, + hash_count: 0 + } + } + /// Get the backing database. pub fn db(&'db self) -> &'db T { self.db @@ -363,7 +379,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// and removing the old. fn set_root_rlp(&mut self, root_data: &[u8]) { self.db.kill(&self.root); - self.root = self.db.insert(root_data); + *self.root = self.db.insert(root_data); self.hash_count += 1; trace!("set_root_rlp {:?} {:?}", root_data.pretty(), self.root); } @@ -944,8 +960,8 @@ mod tests { } } - fn populate_trie<'db, T>(db: &'db mut T, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { - let mut t = TrieDB::new(db); + fn populate_trie<'db, T>(db: &'db mut T, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { + let mut t = TrieDB::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -1005,7 +1021,8 @@ mod tests { let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); - let mut memtrie = populate_trie(&mut memdb, &x); + let mut root = H256::new(); + let mut memtrie = populate_trie(&mut memdb, &mut root, &x); if *memtrie.root() != real || !memtrie.db_items_remaining().is_empty() { println!("TRIE MISMATCH"); println!(""); @@ -1035,7 +1052,8 @@ mod tests { #[test] fn init() { let mut memdb = MemoryDB::new(); - let t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let t = TrieDB::new(&mut memdb, &mut root); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } @@ -1043,7 +1061,8 @@ mod tests { #[test] fn insert_on_empty() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1053,13 +1072,15 @@ mod tests { let big_value = b"00000000000000000000000000000000"; let mut memdb = MemoryDB::new(); - let mut t1 = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t1 = TrieDB::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); trace!("keys remaining {:?}", t1.db_items_remaining()); assert!(t1.db_items_remaining().is_empty()); let mut memdb2 = MemoryDB::new(); - let mut t2 = TrieDB::new(&mut memdb2); + let mut root2 = H256::new(); + let mut t2 = TrieDB::new(&mut memdb2, &mut root2); t2.insert(&[0x01], &big_value.to_vec()); t2.insert(&[0x01, 0x23], &big_value.to_vec()); t2.insert(&[0x01, 0x34], &big_value.to_vec()); @@ -1074,7 +1095,8 @@ mod tests { #[test] fn insert_replace_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1083,7 +1105,8 @@ mod tests { #[test] fn insert_make_branch_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ @@ -1095,7 +1118,8 @@ mod tests { #[test] fn insert_into_branch_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1109,7 +1133,8 @@ mod tests { #[test] fn insert_value_into_branch_root() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[], &[0x0]); assert_eq!(*t.root(), trie_root(vec![ @@ -1121,7 +1146,8 @@ mod tests { #[test] fn insert_split_leaf() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); assert_eq!(*t.root(), trie_root(vec![ @@ -1133,7 +1159,8 @@ mod tests { #[test] fn insert_split_extenstion() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]); t.insert(&[0x01, 0xf3, 0x45], &[0x02]); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); @@ -1150,7 +1177,8 @@ mod tests { let big_value1 = b"11111111111111111111111111111111"; let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0); t.insert(&[0x11u8, 0x23], big_value1); assert_eq!(*t.root(), trie_root(vec![ @@ -1164,7 +1192,8 @@ mod tests { let big_value = b"00000000000000000000000000000000"; let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value); t.insert(&[0x11u8, 0x23], big_value); assert_eq!(*t.root(), trie_root(vec![ @@ -1223,14 +1252,16 @@ mod tests { #[test] fn test_at_empty() { let mut memdb = MemoryDB::new(); - let t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let t = TrieDB::new(&mut memdb, &mut root); assert_eq!(t.at(&[0x5]), None); } #[test] fn test_at_one() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); } @@ -1238,7 +1269,8 @@ mod tests { #[test] fn test_at_three() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1251,7 +1283,8 @@ mod tests { #[test] fn test_print_trie() { let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); @@ -1272,11 +1305,13 @@ mod tests { } let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); - let memtrie = populate_trie(&mut memdb, &x); + let mut root = H256::new(); + let memtrie = populate_trie(&mut memdb, &mut root, &x); let mut y = x.clone(); y.sort_by(|ref a, ref b| a.0.cmp(&b.0)); let mut memdb2 = MemoryDB::new(); - let memtrie_sorted = populate_trie(&mut memdb2, &y); + let mut root2 = H256::new(); + let memtrie_sorted = populate_trie(&mut memdb2, &mut root2, &y); if *memtrie.root() != real || *memtrie_sorted.root() != real { println!("TRIE MISMATCH"); println!(""); @@ -1303,7 +1338,8 @@ mod tests { println!("file: {}", file); let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for operation in input.into_iter() { match operation { trie::Operation::Insert(key, value) => t.insert(&key, &value), @@ -1314,4 +1350,18 @@ mod tests { assert_eq!(*t.root(), H256::from_slice(&output)); }); } + + #[test] + fn test_trie_existing() { + let mut root = H256::new(); + let mut db = MemoryDB::new(); + { + let mut t = TrieDB::new(&mut db, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } + + { + let _ = TrieDB::new_pre_existing(&mut db, &mut root); + } + } } From f19d044e7ce5ef286f9d722004132067bd038103 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 03:49:15 +0100 Subject: [PATCH 251/379] renamed new_pre_existing to new_existing --- src/trie.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 2909e1314..94ca8c7d7 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -329,7 +329,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist - pub fn new_pre_existing(db: &'db mut T, root: &'db mut H256) -> Self { + pub fn new_existing(db: &'db mut T, root: &'db mut H256) -> Self { assert!(db.exists(root)); TrieDB { db: db, @@ -1361,7 +1361,7 @@ mod tests { } { - let _ = TrieDB::new_pre_existing(&mut db, &mut root); + let _ = TrieDB::new_existing(&mut db, &mut root); } } } From 45ee74868a10840513ac78dcd01c843324b35e06 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 11:53:44 +0100 Subject: [PATCH 252/379] fixed trie benches --- benches/trie.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index c12a8f752..65df58309 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -54,7 +54,8 @@ fn trie_insertions_six_high(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -88,7 +89,8 @@ fn trie_insertions_six_mid(b: &mut Bencher) { } b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -124,7 +126,8 @@ fn trie_insertions_random_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -160,7 +163,8 @@ fn trie_insertions_six_low(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); - let mut t = TrieDB::new(&mut memdb); + let mut root = H256::new(); + let mut t = TrieDB::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } From 16cadf1a64abf36d43aefa0ce5ee6ad35e15dbab Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 11 Dec 2015 14:40:28 +0100 Subject: [PATCH 253/379] hash for uint --- src/uint.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/uint.rs b/src/uint.rs index 234867ea9..389894c68 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -25,6 +25,7 @@ use std::fmt; use std::cmp::*; use std::ops::*; use std::str::FromStr; +use std::hash::{Hash, Hasher}; use rustc_serialize::hex::{FromHex, FromHexError}; macro_rules! impl_map_from { @@ -350,6 +351,13 @@ macro_rules! construct_uint { Ok(()) } } + + impl Hash for $name { + fn hash(&self, state: &mut H) where H: Hasher { + unsafe { state.write(::std::slice::from_raw_parts(self.0.as_ptr() as *mut u8, self.0.len() * 8)); } + state.finish(); + } + } ); } From 862702b9c731df55b1dc0dd334830936badc30f7 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Dec 2015 06:04:24 +0100 Subject: [PATCH 254/379] implemented rem, lenient from_str, from_dec_str for uint --- src/uint.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index 389894c68..d0bb55608 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -28,6 +28,11 @@ use std::str::FromStr; use std::hash::{Hash, Hasher}; use rustc_serialize::hex::{FromHex, FromHexError}; +pub trait FromDecStr: Sized { + type Err; + fn from_dec_str(value: &str) -> Result; +} + macro_rules! impl_map_from { ($thing:ident, $from:ty, $to:ty) => { impl From<$from> for $thing { @@ -139,8 +144,13 @@ macro_rules! construct_uint { type Err = FromHexError; fn from_str(value: &str) -> Result<$name, Self::Err> { - let bytes: &[u8] = &try!(value.from_hex()); - Ok(From::from(bytes)) + let bytes: Vec = match value.len() % 2 == 0 { + true => try!(value.from_hex()), + false => try!(("0".to_string() + value).from_hex()) + }; + + let bytes_ref: &[u8] = &bytes; + Ok(From::from(bytes_ref)) } } @@ -222,6 +232,15 @@ macro_rules! construct_uint { } } + impl Rem<$name> for $name { + type Output = $name; + + fn rem(self, other: $name) -> $name { + let times = self / other; + self - (times * other) + } + } + impl BitAnd<$name> for $name { type Output = $name; @@ -358,9 +377,21 @@ macro_rules! construct_uint { state.finish(); } } + + impl FromDecStr for $name { + type Err = FromHexError; + + /// TODO: optimize, throw appropriate err + fn from_dec_str(value: &str) -> Result { + let ten = $name::from(10u64); + Ok(value.bytes().map(|b| b - 48).fold($name::from(0u64), | acc, c | acc * ten + $name::from(c) )) + } + } + ); } +construct_uint!(U512, 8); construct_uint!(U256, 4); construct_uint!(U128, 2); @@ -374,6 +405,10 @@ impl From for U256 { } } + + + + pub const ZERO_U256: U256 = U256([0x00u64; 4]); pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); @@ -381,6 +416,7 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); #[cfg(test)] mod tests { use uint::U256; + use uint::FromDecStr; use std::str::FromStr; #[test] @@ -565,5 +601,17 @@ mod tests { assert_eq!(U256::from(10u64) / U256::from(2u64), U256::from(5u64)); assert_eq!(U256::from(10u64) / U256::from(3u64), U256::from(3u64)); } + + #[test] + fn uint256_rem() { + assert_eq!(U256::from(10u64) % U256::from(1u64), U256::from(0u64)); + assert_eq!(U256::from(10u64) % U256::from(3u64), U256::from(1u64)); + } + + #[test] + fn uint256_from_dec_str() { + assert_eq!(U256::from_dec_str("10").unwrap(), U256::from(10u64)); + assert_eq!(U256::from_dec_str("1024").unwrap(), U256::from(1024u64)); + } } From 38f813a5e641c0183de1e0d420794bacf956473c Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Dec 2015 13:32:42 +0100 Subject: [PATCH 255/379] fast mul by 10 --- src/uint.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index d0bb55608..09b654270 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -383,8 +383,13 @@ macro_rules! construct_uint { /// TODO: optimize, throw appropriate err fn from_dec_str(value: &str) -> Result { - let ten = $name::from(10u64); - Ok(value.bytes().map(|b| b - 48).fold($name::from(0u64), | acc, c | acc * ten + $name::from(c) )) + Ok(value.bytes() + .map(|b| b - 48) + .fold($name::from(0u64), | acc, c | + // fast multiplication by 10 + // (acc << 3) + (acc << 1) => acc * 10 + (acc << 3) + (acc << 1) + $name::from(c) + )) } } From 6bc56ad004d4a982f27cf8a74c38c84726482dd2 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 12 Dec 2015 15:57:08 +0100 Subject: [PATCH 256/379] added raw method to RlpStream. usefull for quicklookup --- src/rlp/rlpstream.rs | 4 ++++ src/rlp/traits.rs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 301a39053..43ae83a9a 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -101,6 +101,10 @@ impl Stream for RlpStream { self.unfinished_lists.len() == 0 } + fn raw(&self) -> &[u8] { + &self.encoder.bytes + } + fn out(self) -> Vec { match self.is_finished() { true => self.encoder.out().to_vec(), diff --git a/src/rlp/traits.rs b/src/rlp/traits.rs index 1127ca3db..4f9ad09e5 100644 --- a/src/rlp/traits.rs +++ b/src/rlp/traits.rs @@ -282,6 +282,8 @@ pub trait Stream: Sized { /// } fn is_finished(&self) -> bool; + fn raw(&self) -> &[u8]; + /// Streams out encoded bytes. /// /// panic! if stream is not finished. From d94c55133bfd2fae0c78e64c396833baeb2816b7 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 13 Dec 2015 19:36:49 +0100 Subject: [PATCH 257/379] Trie takes a reference to HashDB. --- src/trie.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/trie.rs b/src/trie.rs index 94ca8c7d7..ad70a30a1 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -299,8 +299,8 @@ impl <'a>Node<'a> { /// assert!(t.db_items_remaining().is_empty()); /// } /// ``` -pub struct TrieDB<'db, T> where T: 'db + HashDB { - db: &'db mut T, +pub struct TrieDB<'db> { + db: &'db mut HashDB, root: &'db mut H256, pub hash_count: usize, } @@ -311,12 +311,12 @@ enum MaybeChanged<'a> { Changed(Bytes), } -impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { +impl<'db> TrieDB<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. - pub fn new(db: &'db mut T, root: &'db mut H256) -> Self { - let mut r = TrieDB{ + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + let mut r = TrieDB{ db: db, root: root, hash_count: 0 @@ -329,7 +329,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist - pub fn new_existing(db: &'db mut T, root: &'db mut H256) -> Self { + pub fn new_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { assert!(db.exists(root)); TrieDB { db: db, @@ -339,7 +339,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { } /// Get the backing database. - pub fn db(&'db self) -> &'db T { + pub fn db(&'db self) -> &'db HashDB { self.db } @@ -364,7 +364,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { /// Determine occurances of items in the backing database which are not related to this /// trie. pub fn db_items_remaining(&self) -> HashMap { - let mut ret = self.db().keys(); + let mut ret = self.db.keys(); for (k, v) in Self::to_map(self.keys()).into_iter() { let keycount = *ret.get(&k).unwrap_or(&0); match keycount == v as i32 { @@ -892,7 +892,7 @@ impl<'db, T> TrieDB<'db, T> where T: 'db + HashDB { } } -impl<'db, T> Trie for TrieDB<'db, T> where T: 'db + HashDB { +impl<'db> Trie for TrieDB<'db> { fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { @@ -915,7 +915,7 @@ impl<'db, T> Trie for TrieDB<'db, T> where T: 'db + HashDB { } } -impl<'db, T> fmt::Debug for TrieDB<'db, T> where T: 'db + HashDB { +impl<'db> fmt::Debug for TrieDB<'db> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); @@ -960,7 +960,7 @@ mod tests { } } - fn populate_trie<'db, T>(db: &'db mut T, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db, T> where T: 'db + HashDB { + fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db> { let mut t = TrieDB::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; @@ -970,7 +970,7 @@ mod tests { t } - fn unpopulate_trie<'a, 'db, T>(t: &mut TrieDB<'db, T>, v: &Vec<(Vec, Vec)>) where T: 'db + HashDB { + fn unpopulate_trie<'a, 'db>(t: &mut TrieDB<'db>, v: &Vec<(Vec, Vec)>) { for i in v.iter() { let key: &[u8]= &i.0; t.remove(&key); From 83af9402234576bad6bd9300560218a027c0bb78 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 13 Dec 2015 22:44:28 +0100 Subject: [PATCH 258/379] h264 --- src/hash.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index b51bbdacf..e86f8e834 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -43,19 +43,17 @@ macro_rules! impl_hash { impl Deref for $from { type Target = [u8]; + #[inline] fn deref(&self) -> &[u8] { - unsafe { - ::std::slice::from_raw_parts(self.0.as_ptr(), $size) - } + &self.0 } } impl DerefMut for $from { + #[inline] fn deref_mut(&mut self) -> &mut [u8] { - unsafe { - ::std::slice::from_raw_parts_mut(self.0.as_mut_ptr(), $size) - } + &mut self.0 } } @@ -327,6 +325,7 @@ impl_hash!(H64, 8); impl_hash!(H128, 16); impl_hash!(Address, 20); impl_hash!(H256, 32); +impl_hash!(H264, 33); impl_hash!(H512, 64); impl_hash!(H520, 65); impl_hash!(H1024, 128); From de4a227a477216bc6a51c7c59a59951791b6282c Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 14 Dec 2015 11:56:11 +0100 Subject: [PATCH 259/379] decoding fixed sized arrays --- src/rlp/errors.rs | 1 + src/rlp/tests.rs | 8 ++++++++ src/rlp/untrusted_rlp.rs | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/rlp/errors.rs b/src/rlp/errors.rs index ada3c2a47..4ee41a2ce 100644 --- a/src/rlp/errors.rs +++ b/src/rlp/errors.rs @@ -8,6 +8,7 @@ pub enum DecoderError { RlpIsTooShort, RlpExpectedToBeList, RlpExpectedToBeData, + RlpIncorrectListLen } impl StdError for DecoderError { diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs index e44953e07..b4a60a3de 100644 --- a/src/rlp/tests.rs +++ b/src/rlp/tests.rs @@ -343,3 +343,11 @@ fn test_rlp_json() { }); } +#[test] +fn test_decoding_array() { + let v = vec![5u16, 2u16]; + let res = rlp::encode(&v); + let arr: [u16; 2] = rlp::decode(&res); + assert_eq!(arr[0], 5); + assert_eq!(arr[1], 2); +} diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index ec68fce99..90bfef96c 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -352,3 +352,38 @@ impl Decodable for Option where T: Decodable { }) } } + +macro_rules! impl_array_decodable { + ($index_type:ty, $len:expr ) => ( + impl Decodable for [T; $len] where T: Decodable { + fn decode(decoder: &D) -> Result where D: Decoder { + decoder.read_list(| decoders | { + let mut result: [T; $len] = unsafe { ::std::mem::uninitialized() }; + if decoders.len() != $len { + return Err(DecoderError::RlpIncorrectListLen); + } + + for i in 0..decoders.len() { + result[i] = try!(T::decode(&decoders[i])); + } + + Ok(result) + }) + } + } + ) +} + +macro_rules! impl_array_decodable_recursive { + ($index_type:ty, ) => (); + ($index_type:ty, $len:expr, $($more:expr,)*) => ( + impl_array_decodable!($index_type, $len); + impl_array_decodable_recursive!($index_type, $($more,)*); + ); +} + +impl_array_decodable_recursive!( + u8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 40, 48, 56, 64, 72, 96, 128, 160, 192, 224, +); From 9231bc66221626c387b019659c84e1905e3f4f5a Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 14 Dec 2015 12:03:48 +0100 Subject: [PATCH 260/379] read_list -> as_list --- src/rlp/traits.rs | 5 ++--- src/rlp/untrusted_rlp.rs | 33 +++++++++++++++------------------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/rlp/traits.rs b/src/rlp/traits.rs index 4f9ad09e5..067c438bf 100644 --- a/src/rlp/traits.rs +++ b/src/rlp/traits.rs @@ -1,11 +1,10 @@ use rlp::DecoderError; -pub trait Decoder { +pub trait Decoder: Sized { fn read_value(&self, f: F) -> Result where F: FnOnce(&[u8]) -> Result; - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result; + fn as_list(&self) -> Result, DecoderError>; } pub trait Decodable: Sized { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 90bfef96c..a8cecf09f 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -305,13 +305,11 @@ impl<'a> Decoder for BasicDecoder<'a> { } } - fn read_list(&self, f: F) -> Result - where F: FnOnce(&[Self]) -> Result { - + fn as_list(&self) -> Result, DecoderError> { let v: Vec> = self.rlp.iter() .map(| i | BasicDecoder::new(i)) .collect(); - f(&v) + Ok(v) } } @@ -325,9 +323,8 @@ impl Decodable for T where T: FromBytes { impl Decodable for Vec where T: Decodable { fn decode(decoder: &D) -> Result where D: Decoder { - decoder.read_list(| decoders | { - decoders.iter().map(|d| T::decode(d)).collect() - }) + let decoders = try!(decoder.as_list()); + decoders.iter().map(|d| T::decode(d)).collect() } } @@ -357,18 +354,18 @@ macro_rules! impl_array_decodable { ($index_type:ty, $len:expr ) => ( impl Decodable for [T; $len] where T: Decodable { fn decode(decoder: &D) -> Result where D: Decoder { - decoder.read_list(| decoders | { - let mut result: [T; $len] = unsafe { ::std::mem::uninitialized() }; - if decoders.len() != $len { - return Err(DecoderError::RlpIncorrectListLen); - } - - for i in 0..decoders.len() { - result[i] = try!(T::decode(&decoders[i])); - } + let decoders = try!(decoder.as_list()); - Ok(result) - }) + let mut result: [T; $len] = unsafe { ::std::mem::uninitialized() }; + if decoders.len() != $len { + return Err(DecoderError::RlpIncorrectListLen); + } + + for i in 0..decoders.len() { + result[i] = try!(T::decode(&decoders[i])); + } + + Ok(result) } } ) From 1e6694ec7f48e4ba5688516a3a767fd6b15536af Mon Sep 17 00:00:00 2001 From: debris Date: Tue, 15 Dec 2015 16:36:38 +0100 Subject: [PATCH 261/379] removed unused stuff --- src/db.rs | 0 src/lib.rs | 1 - src/uint.rs | 5 ----- 3 files changed, 6 deletions(-) delete mode 100644 src/db.rs diff --git a/src/db.rs b/src/db.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lib.rs b/src/lib.rs index 12998529a..24b404a88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,6 @@ pub mod uint; pub mod bytes; pub mod rlp; pub mod vector; -pub mod db; pub mod sha3; pub mod hashdb; pub mod memorydb; diff --git a/src/uint.rs b/src/uint.rs index 09b654270..88ed49712 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -396,7 +396,6 @@ macro_rules! construct_uint { ); } -construct_uint!(U512, 8); construct_uint!(U256, 4); construct_uint!(U128, 2); @@ -410,10 +409,6 @@ impl From for U256 { } } - - - - pub const ZERO_U256: U256 = U256([0x00u64; 4]); pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); From ca1a6bd7912e77901c74abc9ebf45e875dd53d31 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 16:23:08 +0100 Subject: [PATCH 262/379] heapsize && squeeze --- Cargo.toml | 1 + src/heapsizeof.rs | 5 +++++ src/lib.rs | 6 ++++++ src/squeeze.rs | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) create mode 100644 src/heapsizeof.rs create mode 100644 src/squeeze.rs diff --git a/Cargo.toml b/Cargo.toml index 13295f766..4d23f49f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" elastic-array = "0.4" +heapsize = "0.2" [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/heapsizeof.rs b/src/heapsizeof.rs new file mode 100644 index 000000000..c6d4cace4 --- /dev/null +++ b/src/heapsizeof.rs @@ -0,0 +1,5 @@ +use uint::*; +use hash::*; + +known_heap_size!(0, H32, H64, H128, Address, H256, H264, H512, H520, H1024, H2048); +known_heap_size!(0, U128, U256); diff --git a/src/lib.rs b/src/lib.rs index 24b404a88..9120f0977 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,10 +32,14 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; + +#[macro_use] +extern crate heapsize; #[macro_use] extern crate log; #[macro_use] extern crate lazy_static; + extern crate env_logger; extern crate time; @@ -60,5 +64,7 @@ pub mod crypto; pub mod triehash; pub mod trie; pub mod nibbleslice; +pub mod heapsizeof; +pub mod squeeze; //pub mod network; diff --git a/src/squeeze.rs b/src/squeeze.rs new file mode 100644 index 000000000..220c61800 --- /dev/null +++ b/src/squeeze.rs @@ -0,0 +1,33 @@ +use std::collections::HashMap; +use std::hash::Hash; +use heapsize::HeapSizeOf; + +/// Should be used to squeeze collections to certain size in bytes +trait Squeeze { + fn squeeze(&mut self, size: usize); +} + +impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: HeapSizeOf { + fn squeeze(&mut self, size: usize) { + if self.len() == 0 { + return + } + + let size_of_entry = self.heap_size_of_children() / self.capacity(); + let mut shrinked_size = size_of_entry * self.len(); + + while self.len() > 0 || shrinked_size > size { + // could be optimized + let key = self.keys().next().unwrap().clone(); + self.remove(&key); + shrinked_size -= size_of_entry; + } + + self.shrink_to_fit(); + + // if we havent shrinked enough, squeeze again + if self.heap_size_of_children() > size { + self.squeeze(size); + } + } +} From 56de081381f18e35bcc10c38858f13cd0ffc9768 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 16:27:27 +0100 Subject: [PATCH 263/379] squeeze is pub --- src/squeeze.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/squeeze.rs b/src/squeeze.rs index 220c61800..ddec2a3a9 100644 --- a/src/squeeze.rs +++ b/src/squeeze.rs @@ -3,7 +3,7 @@ use std::hash::Hash; use heapsize::HeapSizeOf; /// Should be used to squeeze collections to certain size in bytes -trait Squeeze { +pub trait Squeeze { fn squeeze(&mut self, size: usize); } From 926efb6fc8c5eff915201bb01fe7f8435e4efc2f Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 16 Dec 2015 17:12:20 +0100 Subject: [PATCH 264/379] docs, tests and bugfixes for squeeze --- src/squeeze.rs | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/squeeze.rs b/src/squeeze.rs index ddec2a3a9..e81a13793 100644 --- a/src/squeeze.rs +++ b/src/squeeze.rs @@ -1,3 +1,35 @@ +//! Helper module that should be used to randomly squeeze +//! caches to a given size in bytes +//! +//! ``` +//! extern crate heapsize; +//! extern crate ethcore_util as util; +//! use std::collections::HashMap; +//! use std::mem::size_of; +//! use heapsize::HeapSizeOf; +//! use util::squeeze::Squeeze; +//! +//! fn main() { +//! let initial_size = 60; +//! let mut map: HashMap = HashMap::with_capacity(initial_size); +//! assert!(map.capacity() >= initial_size); +//! for i in 0..initial_size { +//! map.insert(i as u8, i as u8); +//! } +//! +//! assert_eq!(map.heap_size_of_children(), map.capacity() * 2 * size_of::()); +//! assert_eq!(map.len(), initial_size); +//! let initial_heap_size = map.heap_size_of_children(); +//! +//! // squeeze it to size of key and value +//! map.squeeze(2 * size_of::()); +//! assert_eq!(map.len(), 1); +//! +//! // its likely that heap size was reduced, but we can't be 100% sure +//! assert!(initial_heap_size >= map.heap_size_of_children()); +//! } +//! ``` + use std::collections::HashMap; use std::hash::Hash; use heapsize::HeapSizeOf; @@ -14,9 +46,10 @@ impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: } let size_of_entry = self.heap_size_of_children() / self.capacity(); - let mut shrinked_size = size_of_entry * self.len(); + let all_entries = size_of_entry * self.len(); + let mut shrinked_size = all_entries; - while self.len() > 0 || shrinked_size > size { + while self.len() > 0 && shrinked_size > size { // could be optimized let key = self.keys().next().unwrap().clone(); self.remove(&key); @@ -25,9 +58,10 @@ impl Squeeze for HashMap where K: Eq + Hash + Clone + HeapSizeOf, T: self.shrink_to_fit(); - // if we havent shrinked enough, squeeze again - if self.heap_size_of_children() > size { + // if we squeezed something, but not enough, squeeze again + if all_entries != shrinked_size && self.heap_size_of_children() > size { self.squeeze(size); } } } + From a9bd050d2ff563d915bf5f9eaa935561d9bd5328 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 17 Dec 2015 11:42:30 +0100 Subject: [PATCH 265/379] Net service --- Cargo.toml | 2 +- src/network/connection.rs | 2 +- src/network/handshake.rs | 35 ++----- src/network/host.rs | 211 ++++++++++++++++++++++++++++++++------ src/network/mod.rs | 23 ++++- src/network/service.rs | 52 ++++++++++ src/network/session.rs | 132 +++++++++++++++++++----- 7 files changed, 367 insertions(+), 90 deletions(-) create mode 100644 src/network/service.rs diff --git a/Cargo.toml b/Cargo.toml index d836c093a..fe56748d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ log = "0.3" env_logger = "0.3" rustc-serialize = "0.3" arrayvec = "0.3" -mio = "0.4.4" +mio = "0.5.*" rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" diff --git a/src/network/connection.rs b/src/network/connection.rs index 93a92daa7..65460f1bd 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -120,7 +120,7 @@ impl Connection { pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", self.token); self.interest.insert(EventSet::readable()); - event_loop.register_opt(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { + event_loop.register(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { error!("Failed to reregister {:?}, {:?}", self.token, e); Err(e) }) diff --git a/src/network/handshake.rs b/src/network/handshake.rs index 788b7c1f7..db1511cd0 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -4,17 +4,15 @@ use hash::*; use bytes::Bytes; use crypto::*; use crypto; -use network::connection::{Connection, WriteStatus}; +use network::connection::{Connection}; use network::host::{NodeId, Host, HostInfo}; use network::Error; -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Debug)] enum HandshakeState { New, ReadingAuth, - WritingAuth, ReadingAck, - WritingAck, StartSession, } @@ -89,7 +87,7 @@ impl Handshake { None => {} }; }, - _ => { panic!("Unexpected state") } + _ => { panic!("Unexpected state"); } } if self.state != HandshakeState::StartSession { try!(self.connection.reregister(event_loop)); @@ -99,27 +97,7 @@ impl Handshake { pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); - match self.state { - HandshakeState::WritingAuth => { - match try!(self.connection.writable()) { - WriteStatus::Complete => { - self.connection.expect(ACK_PACKET_SIZE); - self.state = HandshakeState::ReadingAck; - }, - _ => {} - }; - }, - HandshakeState::WritingAck => { - match try!(self.connection.writable()) { - WriteStatus::Complete => { - self.connection.expect(32); - self.state = HandshakeState::StartSession; - }, - _ => {} - }; - }, - _ => { panic!("Unexpected state") } - } + try!(self.connection.writable()); if self.state != HandshakeState::StartSession { try!(self.connection.reregister(event_loop)); } @@ -185,7 +163,8 @@ impl Handshake { let message = try!(crypto::ecies::encrypt(&self.id, &data)); self.auth_cipher = message.clone(); self.connection.send(message); - self.state = HandshakeState::WritingAuth; + self.connection.expect(ACK_PACKET_SIZE); + self.state = HandshakeState::ReadingAck; Ok(()) } @@ -203,7 +182,7 @@ impl Handshake { let message = try!(crypto::ecies::encrypt(&self.id, &data)); self.ack_cipher = message.clone(); self.connection.send(message); - self.state = HandshakeState::WritingAck; + self.state = HandshakeState::StartSession; Ok(()) } } diff --git a/src/network/host.rs b/src/network/host.rs index e35314d88..6c37bb1d3 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -13,15 +13,17 @@ use crypto::*; use rlp::*; use time::Tm; use network::handshake::Handshake; -use network::session::Session; -use network::Error; +use network::session::{Session, SessionData}; +use network::{Error, ProtocolHandler}; const DEFAULT_PORT: u16 = 30304; const MAX_CONNECTIONS: usize = 1024; +const MAX_USER_TIMERS: usize = 32; const IDEAL_PEERS:u32 = 10; pub type NodeId = H512; +type TimerFun = Fn(&mut HostIo) -> bool + Send; #[derive(Debug)] struct NetworkConfiguration { @@ -141,33 +143,100 @@ const NODETABLE_MAINTAIN: usize = 5; const NODETABLE_DISCOVERY: usize = 6; const FIRST_CONNECTION: usize = 7; const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; +const USER_TIMER: usize = LAST_CONNECTION; +const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; + +pub type PacketId = u32; +pub type ProtocolId = &'static str; pub enum HostMessage { - Shutdown + Shutdown, + AddHandler { + handler: Box, + protocol: ProtocolId, + versions: Vec, + }, + Send { + peer: PeerId, + packet_id: PacketId, + protocol: ProtocolId, + data: Vec, + }, + AddTimer { + handler: Box, + delay: u32, + protocol: ProtocolId, + } } +pub type PeerId = u32; + #[derive(Debug, PartialEq, Eq)] pub struct CapabilityInfo { - pub protocol: String, - pub version: u32, + pub protocol: ProtocolId, + pub version: u8, + pub packet_count: u8, } impl Encodable for CapabilityInfo { fn encode(&self, encoder: &mut E) -> () where E: Encoder { encoder.emit_list(|e| { self.protocol.encode(e); - self.version.encode(e); + (self.version as u32).encode(e); }); } } -impl Decodable for CapabilityInfo { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { - Ok(CapabilityInfo { - protocol: try!(String::decode_untrusted(&try!(rlp.at(0)))), - version: try!(u32::decode_untrusted(&try!(rlp.at(1)))), - }) +pub struct HostIo<'s> { + protocol: ProtocolId, + session: Option<&'s mut Session>, + event_loop: &'s mut EventLoop, + channel: &'s mut Sender +} + +impl<'s> HostIo<'s> { + fn new(protocol: ProtocolId, session: Option<&'s mut Session>, event_loop: &'s mut EventLoop, channel: &'s mut Sender) -> HostIo<'s> { + HostIo { + protocol: protocol, + session: session, + event_loop: event_loop, + channel: channel, + } } + + pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), Error> { + try!(self.channel.send(HostMessage::Send { + peer: peer, + packet_id: packet_id, + protocol: self.protocol, + data: data + })); + Ok(()) + } + + pub fn respond(&mut self, packet_id: PacketId, data: &[u8]) -> Result<(), Error> { + match self.session.as_mut() { + Some(session) => session.send_packet(self.protocol, packet_id as u8, data), + None => { + panic!("Respond: Session does not exist") + } + } + } + + fn register_timer(&mut self, ms: u32, handler: Box) -> Result<(), Error>{ + try!(self.channel.send(HostMessage::AddTimer { + delay: ms, + handler: handler, + protocol: self.protocol, + })); + Ok(()) + } +} + +struct UserTimer { + handler: Box, + protocol: ProtocolId, + delay: u32, } pub struct HostInfo { @@ -201,16 +270,18 @@ enum ConnectionEntry { pub struct Host { info: HostInfo, - sender: Sender, udp_socket: UdpSocket, listener: TcpListener, connections: Slab, + timers: Slab, nodes: HashMap, + handlers: HashMap>, idle_timeout: Timeout, + channel: Sender, } impl Host { - pub fn start() { + pub fn start(event_loop: &mut EventLoop) -> Result<(), Error> { let config = NetworkConfiguration::new(); /* match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { @@ -222,19 +293,16 @@ impl Host { let addr = config.listen_address; // Setup the server socket let listener = TcpListener::bind(&addr).unwrap(); - // Create an event loop - let mut event_loop = EventLoop::new().unwrap(); - let sender = event_loop.channel(); // Start listening for incoming connections - event_loop.register_opt(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); + event_loop.register(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); // Setup the client socket //let sock = TcpStream::connect(&addr).unwrap(); // Register the socket - //self.event_loop.register_opt(&sock, CLIENT, EventSet::readable(), PollOpt::edge()).unwrap(); + //self.event_loop.register(&sock, CLIENT, EventSet::readable(), PollOpt::edge()).unwrap(); let idle_timeout = event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay // open the udp socket let udp_socket = UdpSocket::bound(&addr).unwrap(); - event_loop.register_opt(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); + event_loop.register(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); let port = config.listen_address.port(); @@ -246,14 +314,17 @@ impl Host { protocol_version: 4, client_version: "parity".to_string(), listen_port: port, - capabilities: vec![ CapabilityInfo { protocol: "eth".to_string(), version: 63 }], + //capabilities: vec![ CapabilityInfo { protocol: "eth".to_string(), version: 63 }], + capabilities: Vec::new(), }, - sender: sender, udp_socket: udp_socket, listener: listener, connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), + timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), nodes: HashMap::new(), - idle_timeout: idle_timeout + handlers: HashMap::new(), + idle_timeout: idle_timeout, + channel: event_loop.channel(), }; host.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); @@ -263,14 +334,8 @@ impl Host { host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); - event_loop.run(&mut host).unwrap(); - } - - fn stop(&mut self) { - } - - fn have_network(&mut self) -> bool { - true + try!(event_loop.run(&mut host)); + Ok(()) } fn add_node(&mut self, id: &str) { @@ -422,6 +487,8 @@ impl Host { self.start_session(token, event_loop); } } + + fn connection_readable(&mut self, token: Token, event_loop: &mut EventLoop) { let mut kill = false; let mut create_session = false; @@ -435,10 +502,31 @@ impl Host { create_session = h.done(); }, Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.readable(event_loop, &self.info).unwrap_or_else(|e| { + let sd = { s.readable(event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Session read error: {:?}", e); kill = true; - }); + SessionData::None + }) }; + match sd { + SessionData::Ready => { + for (p, h) in self.handlers.iter_mut() { + if s.have_capability(p) { + h.connected(&mut HostIo::new(p, Some(s), event_loop, &mut self.channel), &(token.as_usize() as u32)); + } + } + }, + SessionData::Packet { + data, + protocol, + packet_id, + } => { + match self.handlers.get_mut(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(h) => h.read(&mut HostIo::new(protocol, Some(s), event_loop, &mut self.channel), packet_id, &data[1..]) + } + }, + SessionData::None => {}, + } } _ => { warn!(target: "net", "Received event for unknown connection"); @@ -504,7 +592,62 @@ impl Handler for Host { FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, - _ => panic!("Received unknown timer token"), + USER_TIMER ... LAST_USER_TIMER => { + let timer = self.timers.get_mut(token).expect("Unknown user timer token"); + if (*timer.handler)(&mut HostIo::new(timer.protocol, None, event_loop, &mut self.channel)) { + event_loop.timeout_ms(token, timer.delay as u64).expect("Unable to reregister user timer"); + } + } + _ => panic!("Unknown timer token"), + } + } + + fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { + match msg { + HostMessage::Shutdown => event_loop.shutdown(), + HostMessage::AddHandler { + handler, + protocol, + versions + } => { + self.handlers.insert(protocol, handler); + for v in versions { + self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); + } + }, + HostMessage::Send { + peer, + packet_id, + protocol, + data, + } => { + match self.connections.get_mut(Token(peer as usize)) { + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { + warn!(target: "net", "Send error: {:?}", e); + }); //TODO: don't copy vector data + }, + _ => { + warn!(target: "net", "Send: Peer does not exist"); + } + } + }, + HostMessage::AddTimer { + handler, + delay, + protocol, + } => { + match self.timers.insert(UserTimer { + handler: handler, + delay: delay, + protocol: protocol, + }) { + Ok(token) => { + event_loop.timeout_ms(token, delay as u64).expect("Error registering user timer"); + }, + _ => { panic!("Max timers reached") } + } + } } } } diff --git a/src/network/mod.rs b/src/network/mod.rs index 1199ab967..0b908a417 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -4,6 +4,7 @@ mod connection; mod handshake; mod session; mod discovery; +mod service; #[derive(Debug, Copy, Clone)] pub enum DisconnectReason @@ -31,6 +32,7 @@ pub enum Error { AddressParse(::std::net::AddrParseError), AddressResolve(Option<::std::io::Error>), NodeIdParse(::error::EthcoreError), + PeerNotFound, Disconnect(DisconnectReason) } @@ -61,7 +63,22 @@ impl From<::rlp::DecoderError> for Error { } } -pub fn start_host() -{ - let _ = host::Host::start(); +impl From<::mio::NotifyError> for Error { + fn from(_err: ::mio::NotifyError) -> Error { + Error::Io(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) + } } + +pub type PeerId = host::PeerId; +pub type HandlerIo<'s> = host::HostIo<'s>; + +pub trait ProtocolHandler: Send { + fn initialize(&mut self, io: &mut HandlerIo); + fn read(&mut self, io: &mut HandlerIo, packet_id: u8, data: &[u8]); + fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId); + fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); +} + +pub struct NetworkClient; + + diff --git a/src/network/service.rs b/src/network/service.rs new file mode 100644 index 000000000..1e245c20b --- /dev/null +++ b/src/network/service.rs @@ -0,0 +1,52 @@ +#![allow(dead_code)] //TODO: remove this after everything is done + +use std::thread::{self, JoinHandle}; +use mio::*; +use network::{Error, ProtocolHandler}; +use network::host::{Host, HostMessage, PeerId, PacketId, ProtocolId}; + +pub struct NetworkService { + thread: Option>, + host_channel: Sender +} + +impl NetworkService { + pub fn start() -> Result { + let mut event_loop = EventLoop::new().unwrap(); + let channel = event_loop.channel(); + let thread = thread::spawn(move || { + Host::start(&mut event_loop).unwrap(); //TODO: + }); + Ok(NetworkService { + thread: Some(thread), + host_channel: channel + }) + } + + pub fn send(&mut self, peer: &PeerId, packet_id: PacketId, protocol: ProtocolId, data: &[u8]) -> Result<(), Error> { + try!(self.host_channel.send(HostMessage::Send { + peer: *peer, + packet_id: packet_id, + protocol: protocol, + data: data.to_vec() + })); + Ok(()) + } + + pub fn register_protocol(&mut self, handler: Box, protocol: ProtocolId, versions: &[u8]) -> Result<(), Error> { + try!(self.host_channel.send(HostMessage::AddHandler { + handler: handler, + protocol: protocol, + versions: versions.to_vec(), + })); + Ok(()) + } +} + +impl Drop for NetworkService { + fn drop(&mut self) { + self.host_channel.send(HostMessage::Shutdown).unwrap(); + self.thread.take().unwrap().join().unwrap(); + } +} + diff --git a/src/network/session.rs b/src/network/session.rs index eb6ca6ced..698f8f6ac 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -14,11 +14,44 @@ pub struct Session { had_hello: bool, } +pub enum SessionData { + None, + Ready, + Packet { + data: Vec, + protocol: &'static str, + packet_id: u8, + }, +} + pub struct SessionInfo { pub id: NodeId, pub client_version: String, pub protocol_version: u32, - pub capabilities: Vec, + pub capabilities: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct PeerCapabilityInfo { + pub protocol: String, + pub version: u8, +} + +impl Decodable for PeerCapabilityInfo { + fn decode_untrusted(rlp: &UntrustedRlp) -> Result { + Ok(PeerCapabilityInfo { + protocol: try!(String::decode_untrusted(&try!(rlp.at(0)))), + version: try!(u32::decode_untrusted(&try!(rlp.at(1)))) as u8, + }) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct SessionCapabilityInfo { + pub protocol: &'static str, + pub version: u8, + pub packet_count: u8, + pub id_offset: u8, } const PACKET_HELLO: u8 = 0x80; @@ -50,43 +83,76 @@ impl Session { Ok(session) } - pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { match try!(self.connection.readable(event_loop)) { - Some(data) => { - try!(self.read_packet(data, host)); - }, - None => {} - }; - Ok(()) + Some(data) => self.read_packet(data, host), + None => Ok(SessionData::None) + } } pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { self.connection.writable(event_loop) } - pub fn read_packet(&mut self, packet: Packet, host: &HostInfo) -> Result<(), Error> { - let data = &packet.data; - if data.len() < 2 { + pub fn have_capability(&self, protocol: &str) -> bool { + self.info.capabilities.iter().any(|c| c.protocol == protocol) + } + + pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), Error> { + let mut i = 0usize; + while protocol != self.info.capabilities[i].protocol { + i += 1; + if i == self.info.capabilities.len() { + debug!(target: "net", "Unkown protocol: {:?}", protocol); + return Ok(()) + } + } + let pid = self.info.capabilities[i].id_offset + packet_id; + let mut rlp = RlpStream::new(); + rlp.append(&(pid as u32)); + rlp.append_raw(data, 1); + self.connection.send_packet(&rlp.out()) + } + + fn read_packet(&mut self, packet: Packet, host: &HostInfo) -> Result { + if packet.data.len() < 2 { return Err(Error::BadProtocol); } - let packet_id = data[0]; - let rlp = UntrustedRlp::new(&data[1..]); //TODO: validate rlp expected size + let packet_id = packet.data[0]; if packet_id != PACKET_HELLO && packet_id != PACKET_DISCONNECT && !self.had_hello { return Err(Error::BadProtocol); } match packet_id { - PACKET_HELLO => self.read_hello(&rlp, host), + PACKET_HELLO => { + let rlp = UntrustedRlp::new(&packet.data[1..]); //TODO: validate rlp expected size + try!(self.read_hello(&rlp, host)); + Ok(SessionData::Ready) + } PACKET_DISCONNECT => Err(Error::Disconnect(DisconnectReason::DisconnectRequested)), - PACKET_PING => self.write_pong(), - PACKET_GET_PEERS => Ok(()), //TODO; - PACKET_PEERS => Ok(()), + PACKET_PING => { + try!(self.write_pong()); + Ok(SessionData::None) + } + PACKET_GET_PEERS => Ok(SessionData::None), //TODO; + PACKET_PEERS => Ok(SessionData::None), PACKET_USER ... PACKET_LAST => { - warn!(target: "net", "User packet: {:?}", rlp); - Ok(()) + let mut i = 0usize; + while packet_id < self.info.capabilities[i].id_offset { + i += 1; + if i == self.info.capabilities.len() { + debug!(target: "net", "Unkown packet: {:?}", packet_id); + return Ok(SessionData::None) + } + } + + // map to protocol + let protocol = self.info.capabilities[i].protocol; + let pid = packet_id - self.info.capabilities[i].id_offset; + return Ok(SessionData::Packet { data: packet.data, protocol: protocol, packet_id: pid } ) }, _ => { - debug!(target: "net", "Unkown packet: {:?}", rlp); - Ok(()) + debug!(target: "net", "Unkown packet: {:?}", packet_id); + Ok(SessionData::None) } } } @@ -106,12 +172,24 @@ impl Session { fn read_hello(&mut self, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), Error> { let protocol = try!(u32::decode_untrusted(&try!(rlp.at(0)))); let client_version = try!(String::decode_untrusted(&try!(rlp.at(1)))); - let mut caps: Vec = try!(Decodable::decode_untrusted(&try!(rlp.at(2)))); + let peer_caps: Vec = try!(Decodable::decode_untrusted(&try!(rlp.at(2)))); let id = try!(NodeId::decode_untrusted(&try!(rlp.at(4)))); // Intersect with host capabilities // Leave only highset mutually supported capability version - caps.retain(|c| host.capabilities.contains(&c)); + let mut caps: Vec = Vec::new(); + for hc in host.capabilities.iter() { + if peer_caps.iter().any(|c| c.protocol == hc.protocol && c.version == hc.version) { + caps.push(SessionCapabilityInfo { + protocol: hc.protocol, + version: hc.version, + id_offset: 0, + packet_count: hc.packet_count, + }); + } + } + + caps.retain(|c| host.capabilities.iter().any(|hc| hc.protocol == c.protocol && hc.version == c.version)); let mut i = 0; while i < caps.len() { if caps.iter().any(|c| c.protocol == caps[i].protocol && c.version > caps[i].version) { @@ -122,7 +200,15 @@ impl Session { } } + i = 0; + let mut offset: u8 = PACKET_USER; + while i < caps.len() { + caps[i].id_offset = offset; + offset += caps[i].packet_count; + i += 1; + } trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); + self.info.capabilities = caps; if protocol != host.protocol_version { return Err(self.disconnect(DisconnectReason::UselessPeer)); } From 674c667114370eb2975765bef47362013d308f24 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 17 Dec 2015 12:43:50 +0100 Subject: [PATCH 266/379] Rename Trie to TrieMut in preparation for immutable trie. --- benches/trie.rs | 8 ++--- src/bytes.rs | 12 +++++++ src/hashdb.rs | 12 +++++-- src/trie.rs | 95 +++++++++++++++++++++++++++---------------------- 4 files changed, 78 insertions(+), 49 deletions(-) diff --git a/benches/trie.rs b/benches/trie.rs index 65df58309..8f1a2a42d 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -55,7 +55,7 @@ fn trie_insertions_six_high(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -90,7 +90,7 @@ fn trie_insertions_six_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -127,7 +127,7 @@ fn trie_insertions_random_mid(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } @@ -164,7 +164,7 @@ fn trie_insertions_six_low(b: &mut Bencher) { b.iter(||{ let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for i in d.iter() { t.insert(&i.0, &i.1); } diff --git a/src/bytes.rs b/src/bytes.rs index e81cd8572..d188eb69b 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -54,8 +54,20 @@ impl<'a> fmt::Debug for PrettySlice<'a> { } } +impl<'a> fmt::Display for PrettySlice<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for i in 0..self.0.len() { + try!(write!(f, "{:02x}", self.0[i])); + } + Ok(()) + } +} + pub trait ToPretty { fn pretty(&self) -> PrettySlice; + fn to_hex(&self) -> String { + format!("{}", self.pretty()) + } } impl<'a> ToPretty for &'a [u8] { diff --git a/src/hashdb.rs b/src/hashdb.rs index e63d031e5..0eebe9d9e 100644 --- a/src/hashdb.rs +++ b/src/hashdb.rs @@ -8,6 +8,8 @@ pub trait HashDB { /// Get the keys in the database together with number of underlying references. fn keys(&self) -> HashMap; + /// Deprecated. use `get`. + fn lookup(&self, key: &H256) -> Option<&[u8]>; // TODO: rename to get. /// Look up a given hash into the bytes that hash to it, returning None if the /// hash is not known. /// @@ -23,8 +25,10 @@ pub trait HashDB { /// assert_eq!(m.lookup(&hash).unwrap(), hello_bytes); /// } /// ``` - fn lookup(&self, key: &H256) -> Option<&[u8]>; + fn get(&self, key: &H256) -> Option<&[u8]> { self.lookup(key) } + /// Deprecated. Use `contains`. + fn exists(&self, key: &H256) -> bool; // TODO: rename to contains. /// Check for the existance of a hash-key. /// /// # Examples @@ -43,7 +47,7 @@ pub trait HashDB { /// assert!(!m.exists(&key)); /// } /// ``` - fn exists(&self, key: &H256) -> bool; + fn contains(&self, key: &H256) -> bool { self.exists(key) } /// Insert a datum item into the DB and return the datum's hash for a later lookup. Insertions /// are counted and the equivalent number of `kill()`s must be performed before the data @@ -66,6 +70,8 @@ pub trait HashDB { /// Like `insert()` , except you provide the key and the data is all moved. fn emplace(&mut self, key: H256, value: Bytes); + /// Deprecated - use `remove`. + fn kill(&mut self, key: &H256); // TODO: rename to remove. /// Remove a datum previously inserted. Insertions can be "owed" such that the same number of `insert()`s may /// happen without the data being eventually being inserted into the DB. /// @@ -87,5 +93,5 @@ pub trait HashDB { /// assert_eq!(m.lookup(key).unwrap(), d); /// } /// ``` - fn kill(&mut self, key: &H256); + fn remove(&mut self, key: &H256) { self.kill(key) } } diff --git a/src/trie.rs b/src/trie.rs index ad70a30a1..1716aa56f 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -27,8 +27,11 @@ pub trait Trie { fn contains(&self, key: &[u8]) -> bool; /// What is the value of the given key in this trie? - fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; +} +/// A key-value datastore implemented as a database-backed modified Merkle tree. +pub trait TrieMut: Trie { /// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing /// `key` from the trie. fn insert(&mut self, key: &[u8], value: &[u8]); @@ -287,12 +290,12 @@ impl <'a>Node<'a> { /// fn main() { /// let mut memdb = MemoryDB::new(); /// let mut root = H256::new(); -/// let mut t = TrieDB::new(&mut memdb, &mut root); +/// let mut t = TrieDBMut::new(&mut memdb, &mut root); /// assert!(t.is_empty()); /// assert_eq!(*t.root(), SHA3_NULL_RLP); /// t.insert(b"foo", b"bar"); /// assert!(t.contains(b"foo")); -/// assert_eq!(t.at(b"foo").unwrap(), b"bar"); +/// assert_eq!(t.get(b"foo").unwrap(), b"bar"); /// assert!(t.db_items_remaining().is_empty()); /// t.remove(b"foo"); /// assert!(!t.contains(b"foo")); @@ -300,6 +303,12 @@ impl <'a>Node<'a> { /// } /// ``` pub struct TrieDB<'db> { + db: &'db HashDB, + root: &'db H256, + pub hash_count: usize, +} + +pub struct TrieDBMut<'db> { db: &'db mut HashDB, root: &'db mut H256, pub hash_count: usize, @@ -311,12 +320,12 @@ enum MaybeChanged<'a> { Changed(Bytes), } -impl<'db> TrieDB<'db> { +impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. /// This guarantees the trie is built correctly. pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { - let mut r = TrieDB{ + let mut r = TrieDBMut{ db: db, root: root, hash_count: 0 @@ -331,7 +340,7 @@ impl<'db> TrieDB<'db> { /// Panics, if `root` does not exist pub fn new_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { assert!(db.exists(root)); - TrieDB { + TrieDBMut { db: db, root: root, hash_count: 0 @@ -390,11 +399,11 @@ impl<'db> TrieDB<'db> { for d in journal.0.into_iter() { match d { Operation::Delete(h) => { - trace!("TrieDB::apply --- {:?}", &h); + trace!("TrieDBMut::apply --- {:?}", &h); self.db.kill(&h); }, Operation::New(h, d) => { - trace!("TrieDB::apply +++ {:?} -> {:?}", &h, d.pretty()); + trace!("TrieDBMut::apply +++ {:?} -> {:?}", &h, d.pretty()); self.db.emplace(h, d); self.hash_count += 1; } @@ -475,7 +484,7 @@ impl<'db> TrieDB<'db> { } /// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists. - fn get<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + fn do_lookup<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); self.get_from_node(&root_rlp, key) } @@ -892,17 +901,19 @@ impl<'db> TrieDB<'db> { } } -impl<'db> Trie for TrieDB<'db> { +impl<'db> Trie for TrieDBMut<'db> { fn root(&self) -> &H256 { &self.root } fn contains(&self, key: &[u8]) -> bool { - self.at(key).is_some() + self.get(key).is_some() } - fn at<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { - self.get(&NibbleSlice::new(key)) + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.do_lookup(&NibbleSlice::new(key)) } +} +impl<'db> TrieMut for TrieDBMut<'db> { fn insert(&mut self, key: &[u8], value: &[u8]) { match value.is_empty() { false => self.insert_ns(&NibbleSlice::new(key), value), @@ -915,7 +926,7 @@ impl<'db> Trie for TrieDB<'db> { } } -impl<'db> fmt::Debug for TrieDB<'db> { +impl<'db> fmt::Debug for TrieDBMut<'db> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { try!(writeln!(f, "c={:?} [", self.hash_count)); let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); @@ -960,8 +971,8 @@ mod tests { } } - fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDB<'db> { - let mut t = TrieDB::new(db, root); + fn populate_trie<'db>(db: &'db mut HashDB, root: &'db mut H256, v: &Vec<(Vec, Vec)>) -> TrieDBMut<'db> { + let mut t = TrieDBMut::new(db, root); for i in 0..v.len() { let key: &[u8]= &v[i].0; let val: &[u8] = &v[i].1; @@ -970,7 +981,7 @@ mod tests { t } - fn unpopulate_trie<'a, 'db>(t: &mut TrieDB<'db>, v: &Vec<(Vec, Vec)>) { + fn unpopulate_trie<'a, 'db>(t: &mut TrieDBMut<'db>, v: &Vec<(Vec, Vec)>) { for i in v.iter() { let key: &[u8]= &i.0; t.remove(&key); @@ -1053,7 +1064,7 @@ mod tests { fn init() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let t = TrieDB::new(&mut memdb, &mut root); + let t = TrieDBMut::new(&mut memdb, &mut root); assert_eq!(*t.root(), SHA3_NULL_RLP); assert!(t.is_empty()); } @@ -1062,7 +1073,7 @@ mod tests { fn insert_on_empty() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x01u8, 0x23]) ])); } @@ -1073,14 +1084,14 @@ mod tests { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t1 = TrieDB::new(&mut memdb, &mut root); + let mut t1 = TrieDBMut::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); trace!("keys remaining {:?}", t1.db_items_remaining()); assert!(t1.db_items_remaining().is_empty()); let mut memdb2 = MemoryDB::new(); let mut root2 = H256::new(); - let mut t2 = TrieDB::new(&mut memdb2, &mut root2); + let mut t2 = TrieDBMut::new(&mut memdb2, &mut root2); t2.insert(&[0x01], &big_value.to_vec()); t2.insert(&[0x01, 0x23], &big_value.to_vec()); t2.insert(&[0x01, 0x34], &big_value.to_vec()); @@ -1096,7 +1107,7 @@ mod tests { fn insert_replace_root() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x23], &[0x23u8, 0x45]); assert_eq!(*t.root(), trie_root(vec![ (vec![0x01u8, 0x23], vec![0x23u8, 0x45]) ])); @@ -1106,7 +1117,7 @@ mod tests { fn insert_make_branch_root() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x11u8, 0x23], &[0x11u8, 0x23]); assert_eq!(*t.root(), trie_root(vec![ @@ -1119,7 +1130,7 @@ mod tests { fn insert_into_branch_root() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); @@ -1134,7 +1145,7 @@ mod tests { fn insert_value_into_branch_root() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[], &[0x0]); assert_eq!(*t.root(), trie_root(vec![ @@ -1147,7 +1158,7 @@ mod tests { fn insert_split_leaf() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x01u8, 0x34], &[0x01u8, 0x34]); assert_eq!(*t.root(), trie_root(vec![ @@ -1160,7 +1171,7 @@ mod tests { fn insert_split_extenstion() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01, 0x23, 0x45], &[0x01]); t.insert(&[0x01, 0xf3, 0x45], &[0x02]); t.insert(&[0x01, 0xf3, 0xf5], &[0x03]); @@ -1178,7 +1189,7 @@ mod tests { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value0); t.insert(&[0x11u8, 0x23], big_value1); assert_eq!(*t.root(), trie_root(vec![ @@ -1193,7 +1204,7 @@ mod tests { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], big_value); t.insert(&[0x11u8, 0x23], big_value); assert_eq!(*t.root(), trie_root(vec![ @@ -1253,38 +1264,38 @@ mod tests { fn test_at_empty() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let t = TrieDB::new(&mut memdb, &mut root); - assert_eq!(t.at(&[0x5]), None); + let t = TrieDBMut::new(&mut memdb, &mut root); + assert_eq!(t.get(&[0x5]), None); } #[test] fn test_at_one() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); - assert_eq!(t.at(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); + assert_eq!(t.get(&[0x1, 0x23]).unwrap(), &[0x1u8, 0x23]); } #[test] fn test_at_three() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); t.insert(&[0x81u8, 0x23], &[0x81u8, 0x23]); - assert_eq!(t.at(&[0x01, 0x23]).unwrap(), &[0x01u8, 0x23]); - assert_eq!(t.at(&[0xf1, 0x23]).unwrap(), &[0xf1u8, 0x23]); - assert_eq!(t.at(&[0x81, 0x23]).unwrap(), &[0x81u8, 0x23]); - assert_eq!(t.at(&[0x82, 0x23]), None); + assert_eq!(t.get(&[0x01, 0x23]).unwrap(), &[0x01u8, 0x23]); + assert_eq!(t.get(&[0xf1, 0x23]).unwrap(), &[0xf1u8, 0x23]); + assert_eq!(t.get(&[0x81, 0x23]).unwrap(), &[0x81u8, 0x23]); + assert_eq!(t.get(&[0x82, 0x23]), None); } #[test] fn test_print_trie() { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0x02u8, 0x23], &[0x01u8, 0x23]); t.insert(&[0xf1u8, 0x23], &[0xf1u8, 0x23]); @@ -1339,7 +1350,7 @@ mod tests { let mut memdb = MemoryDB::new(); let mut root = H256::new(); - let mut t = TrieDB::new(&mut memdb, &mut root); + let mut t = TrieDBMut::new(&mut memdb, &mut root); for operation in input.into_iter() { match operation { trie::Operation::Insert(key, value) => t.insert(&key, &value), @@ -1356,12 +1367,12 @@ mod tests { let mut root = H256::new(); let mut db = MemoryDB::new(); { - let mut t = TrieDB::new(&mut db, &mut root); + let mut t = TrieDBMut::new(&mut db, &mut root); t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); } { - let _ = TrieDB::new_existing(&mut db, &mut root); + let _ = TrieDBMut::new_existing(&mut db, &mut root); } } } From 2a5c6665cf40a0cf2696ecab7567f769b7342fb0 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 17 Dec 2015 12:48:47 +0100 Subject: [PATCH 267/379] Trie (immutable). --- src/trie.rs | 179 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/src/trie.rs b/src/trie.rs index 1716aa56f..6c0f6dc78 100644 --- a/src/trie.rs +++ b/src/trie.rs @@ -320,6 +320,185 @@ enum MaybeChanged<'a> { Changed(Bytes), } +impl<'db> TrieDB<'db> { + /// Create a new trie with the backing database `db` and `root` + /// Panics, if `root` does not exist + pub fn new(db: &'db HashDB, root: &'db H256) -> Self { + assert!(db.exists(root)); + TrieDB { + db: db, + root: root, + hash_count: 0 + } + } + + /// Get the backing database. + pub fn db(&'db self) -> &'db HashDB { + self.db + } + + /// Determine all the keys in the backing database that belong to the trie. + pub fn keys(&self) -> Vec { + let mut ret: Vec = Vec::new(); + ret.push(self.root.clone()); + self.accumulate_keys(self.root_node(), &mut ret); + ret + } + + /// Convert a vector of hashes to a hashmap of hash to occurances. + pub fn to_map(hashes: Vec) -> HashMap { + let mut r: HashMap = HashMap::new(); + for h in hashes.into_iter() { + let c = *r.get(&h).unwrap_or(&0); + r.insert(h, c + 1); + } + r + } + + /// Determine occurances of items in the backing database which are not related to this + /// trie. + pub fn db_items_remaining(&self) -> HashMap { + let mut ret = self.db.keys(); + for (k, v) in Self::to_map(self.keys()).into_iter() { + let keycount = *ret.get(&k).unwrap_or(&0); + match keycount == v as i32 { + true => ret.remove(&k), + _ => ret.insert(k, keycount - v as i32), + }; + } + ret + } + + /// Recursion helper for `keys`. + fn accumulate_keys(&self, node: Node, acc: &mut Vec) { + let mut handle_payload = |payload| { + let p = Rlp::new(payload); + if p.is_data() && p.size() == 32 { + acc.push(p.as_val()); + } + + self.accumulate_keys(self.get_node(payload), acc); + }; + + match node { + Node::Extension(_, payload) => handle_payload(payload), + Node::Branch(payloads, _) => for payload in payloads.iter() { handle_payload(payload) }, + _ => {}, + } + } + + /// Get the root node's RLP. + fn root_node(&self) -> Node { + Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) + } + + /// Get the root node as a `Node`. + fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { + Node::decoded(self.get_raw_or_lookup(node)) + } + + /// Indentation helper for `formal_all`. + fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { + for _ in 0..size { + try!(write!(f, " ")); + } + Ok(()) + } + + /// Recursion helper for implementation of formatting trait. + fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { + match node { + Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())), + Node::Extension(ref slice, ref item) => { + try!(write!(f, "'{:?} ", slice)); + try!(self.fmt_all(self.get_node(item), f, deepness)); + }, + Node::Branch(ref nodes, ref value) => { + try!(writeln!(f, "")); + match value { + &Some(v) => { + try!(self.fmt_indent(f, deepness + 1)); + try!(writeln!(f, "=: {:?}", v.pretty())) + }, + &None => {} + } + for i in 0..16 { + match self.get_node(nodes[i]) { + Node::Empty => {}, + n => { + try!(self.fmt_indent(f, deepness + 1)); + try!(write!(f, "'{:x} ", i)); + try!(self.fmt_all(n, f, deepness + 1)); + } + } + } + }, + // empty + Node::Empty => { + try!(writeln!(f, "")); + } + }; + Ok(()) + } + + /// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists. + fn do_lookup<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + self.get_from_node(&root_rlp, key) + } + + /// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no + /// value exists for the key. + /// + /// Note: Not a public API; use Trie trait functions. + fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + match Node::decoded(node) { + Node::Leaf(ref slice, ref value) if key == slice => Some(value), + Node::Extension(ref slice, ref item) if key.starts_with(slice) => { + self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len())) + }, + Node::Branch(ref nodes, value) => match key.is_empty() { + true => value, + false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1)) + }, + _ => None + } + } + + /// Given some node-describing data `node`, return the actual node RLP. + /// This could be a simple identity operation in the case that the node is sufficiently small, but + /// may require a database lookup. + fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { + // check if its sha3 + len + let r = Rlp::new(node); + match r.is_data() && r.size() == 32 { + true => self.db.lookup(&r.as_val::()).expect("Not found!"), + false => node + } + } +} + +impl<'db> Trie for TrieDB<'db> { + fn root(&self) -> &H256 { &self.root } + + fn contains(&self, key: &[u8]) -> bool { + self.get(key).is_some() + } + + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.do_lookup(&NibbleSlice::new(key)) + } +} + +impl<'db> fmt::Debug for TrieDB<'db> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(writeln!(f, "c={:?} [", self.hash_count)); + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); + writeln!(f, "]") + } +} + impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. From 6e4850d4aeb604adf5c17559fc3ab36ddcf1c65c Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 17 Dec 2015 14:05:13 +0100 Subject: [PATCH 268/379] updated for new RLP interface --- src/network/connection.rs | 2 +- src/network/session.rs | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/network/connection.rs b/src/network/connection.rs index 65460f1bd..9fbd4df76 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -253,7 +253,7 @@ impl EncryptedConnection { let length = ((((hdec[0] as u32) << 8) + (hdec[1] as u32)) << 8) + (hdec[2] as u32); let header_rlp = UntrustedRlp::new(&hdec[3..6]); - let protocol_id = try!(u16::decode_untrusted(&try!(header_rlp.at(0)))); + let protocol_id = try!(header_rlp.val_at::(0)); self.payload_len = length; self.protocol_id = protocol_id; diff --git a/src/network/session.rs b/src/network/session.rs index 698f8f6ac..a34f6ff1a 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -38,10 +38,12 @@ pub struct PeerCapabilityInfo { } impl Decodable for PeerCapabilityInfo { - fn decode_untrusted(rlp: &UntrustedRlp) -> Result { + fn decode(decoder: &D) -> Result where D: Decoder { + let c = try!(decoder.as_list()); + let v: u32 = try!(Decodable::decode(&c[1])); Ok(PeerCapabilityInfo { - protocol: try!(String::decode_untrusted(&try!(rlp.at(0)))), - version: try!(u32::decode_untrusted(&try!(rlp.at(1)))) as u8, + protocol: try!(Decodable::decode(&c[0])), + version: v as u8, }) } } @@ -170,10 +172,10 @@ impl Session { } fn read_hello(&mut self, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), Error> { - let protocol = try!(u32::decode_untrusted(&try!(rlp.at(0)))); - let client_version = try!(String::decode_untrusted(&try!(rlp.at(1)))); - let peer_caps: Vec = try!(Decodable::decode_untrusted(&try!(rlp.at(2)))); - let id = try!(NodeId::decode_untrusted(&try!(rlp.at(4)))); + let protocol = try!(rlp.val_at::(0)); + let client_version = try!(rlp.val_at::(1)); + let peer_caps = try!(rlp.val_at::>(2)); + let id = try!(rlp.val_at::(4)); // Intersect with host capabilities // Leave only highset mutually supported capability version From f7ee9436cd9d5248a5acffb7fc8867d6653565d6 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 18 Dec 2015 10:29:06 +0100 Subject: [PATCH 269/379] more convinient creation of uint --- src/uint.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/uint.rs b/src/uint.rs index 88ed49712..b8eccc4cc 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -97,6 +97,16 @@ macro_rules! construct_uint { } } + #[inline] + pub fn zero() -> $name { + From::from(0u64) + } + + #[inline] + pub fn one() -> $name { + From::from(1u64) + } + /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -125,6 +135,21 @@ macro_rules! construct_uint { impl_map_from!($name, u8, u64); impl_map_from!($name, u16, u64); impl_map_from!($name, u32, u64); + impl_map_from!($name, usize, u64); + + impl From for $name { + fn from(value: i64) -> $name { + match value >= 0 { + true => From::from(value as u64), + false => { panic!("Unsigned integer can't be created from negative value"); } + } + } + } + + impl_map_from!($name, i8, i64); + impl_map_from!($name, i16, i64); + impl_map_from!($name, i32, i64); + impl_map_from!($name, isize, i64); impl<'a> From<&'a [u8]> for $name { fn from(bytes: &[u8]) -> $name { From 80cdde29724f86f9309edfd6592f81aeee1fc723 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 19 Dec 2015 12:35:26 +0000 Subject: [PATCH 270/379] Renamed RLP's generic submodules to prevent systematic name clashing. Repotted Trie module. --- src/rlp/mod.rs | 12 +- src/rlp/{errors.rs => rlperrors.rs} | 0 src/rlp/{traits.rs => rlptraits.rs} | 0 src/trie/journal.rs | 76 +++++ src/trie/mod.rs | 11 + src/trie/node.rs | 121 +++++++ src/trie/standardmap.rs | 75 ++++ src/trie/triedb.rs | 225 ++++++++++++ src/{trie.rs => trie/triedbmut.rs} | 513 +--------------------------- src/trie/trietraits.rs | 29 ++ 10 files changed, 557 insertions(+), 505 deletions(-) rename src/rlp/{errors.rs => rlperrors.rs} (100%) rename src/rlp/{traits.rs => rlptraits.rs} (100%) create mode 100644 src/trie/journal.rs create mode 100644 src/trie/mod.rs create mode 100644 src/trie/node.rs create mode 100644 src/trie/standardmap.rs create mode 100644 src/trie/triedb.rs rename src/{trie.rs => trie/triedbmut.rs} (70%) create mode 100644 src/trie/trietraits.rs diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 981e5f973..9c15259bd 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -30,8 +30,8 @@ //! * You want to get view onto rlp-slice. //! * You don't want to decode whole rlp at once. -pub mod errors; -pub mod traits; +pub mod rlptraits; +pub mod rlperrors; pub mod rlp; pub mod untrusted_rlp; pub mod rlpstream; @@ -39,11 +39,15 @@ pub mod rlpstream; #[cfg(test)] mod tests; -pub use self::errors::DecoderError; -pub use self::traits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; +pub use self::rlperrors::DecoderError; +pub use self::rlptraits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; pub use self::rlp::{Rlp, RlpIterator}; pub use self::rlpstream::{RlpStream}; +use super::hash::H256; + +pub const NULL_RLP: [u8; 1] = [0x80; 1]; +pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); /// Shortcut function to decode trusted rlp /// diff --git a/src/rlp/errors.rs b/src/rlp/rlperrors.rs similarity index 100% rename from src/rlp/errors.rs rename to src/rlp/rlperrors.rs diff --git a/src/rlp/traits.rs b/src/rlp/rlptraits.rs similarity index 100% rename from src/rlp/traits.rs rename to src/rlp/rlptraits.rs diff --git a/src/trie/journal.rs b/src/trie/journal.rs new file mode 100644 index 000000000..77efc991c --- /dev/null +++ b/src/trie/journal.rs @@ -0,0 +1,76 @@ +use sha3::*; +use hash::H256; +use bytes::*; +use rlp::*; +use hashdb::*; + +/// Type of operation for the backing database - either a new node or a node deletion. +#[derive(Debug)] +enum Operation { + New(H256, Bytes), + Delete(H256), +} + +pub struct Score { + pub inserts: usize, + pub removes: usize, +} + +/// A journal of operations on the backing database. +#[derive(Debug)] +pub struct Journal (Vec); + +impl Journal { + /// Create a new, empty, object. + pub fn new() -> Journal { Journal(vec![]) } + + /// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal` + /// such that the reference is valid, once applied. + pub fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { + if rlp.len() >= 32 { + let rlp_sha3 = rlp.sha3(); + + trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty()); + out.append(&rlp_sha3); + self.0.push(Operation::New(rlp_sha3, rlp)); + } + else { + trace!("new_node: inline node {:?}", rlp.pretty()); + out.append_raw(&rlp, 1); + } + } + + /// Given the RLP that encodes a now-unused node, leave `journal` in such a state that it is noted. + pub fn delete_node_sha3(&mut self, old_sha3: H256) { + trace!("delete_node: {:?}", old_sha3); + self.0.push(Operation::Delete(old_sha3)); + } + + /// Register an RLP-encoded node for deletion (given a slice), if it needs to be deleted. + pub fn delete_node(&mut self, old: &[u8]) { + let r = Rlp::new(old); + if r.is_data() && r.size() == 32 { + self.delete_node_sha3(r.as_val()); + } + } + + pub fn apply(self, db: &mut HashDB) -> Score { + trace!("applying {:?} changes", self.0.len()); + let mut ret = Score{inserts: 0, removes: 0}; + for d in self.0.into_iter() { + match d { + Operation::Delete(h) => { + trace!("TrieDBMut::apply --- {:?}", &h); + db.remove(&h); + ret.removes += 1; + }, + Operation::New(h, d) => { + trace!("TrieDBMut::apply +++ {:?} -> {:?}", &h, d.pretty()); + db.emplace(h, d); + ret.inserts += 1; + } + } + } + ret + } +} diff --git a/src/trie/mod.rs b/src/trie/mod.rs new file mode 100644 index 000000000..4469926d5 --- /dev/null +++ b/src/trie/mod.rs @@ -0,0 +1,11 @@ +pub mod trietraits; +pub mod standardmap; +pub mod journal; +pub mod node; +pub mod triedb; +pub mod triedbmut; + +pub use self::trietraits::*; +pub use self::standardmap::*; +pub use self::triedbmut::*; +pub use self::triedb::*; diff --git a/src/trie/node.rs b/src/trie/node.rs new file mode 100644 index 000000000..03f35a86d --- /dev/null +++ b/src/trie/node.rs @@ -0,0 +1,121 @@ +use hash::*; +use nibbleslice::*; +use bytes::*; +use rlp::*; +use super::journal::*; + +/// Type of node in the trie and essential information thereof. +#[derive(Eq, PartialEq, Debug)] +pub enum Node<'a> { + Empty, + Leaf(NibbleSlice<'a>, &'a[u8]), + Extension(NibbleSlice<'a>, &'a[u8]), + Branch([&'a[u8]; 16], Option<&'a [u8]>) +} + +impl<'a> Node<'a> { + /// Decode the `node_rlp` and return the Node. + pub fn decoded(node_rlp: &'a [u8]) -> Node<'a> { + let r = Rlp::new(node_rlp); + match r.prototype() { + // either leaf or extension - decode first item with NibbleSlice::??? + // and use is_leaf return to figure out which. + // if leaf, second item is a value (is_data()) + // if extension, second item is a node (either SHA3 to be looked up and + // fed back into this function or inline RLP which can be fed back into this function). + Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { + (slice, true) => Node::Leaf(slice, r.at(1).data()), + (slice, false) => Node::Extension(slice, r.at(1).raw()), + }, + // branch - first 16 are nodes, 17th is a value (or empty). + Prototype::List(17) => { + let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; + for i in 0..16 { + nodes[i] = r.at(i).raw(); + } + Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) + }, + // an empty branch index. + Prototype::Data(0) => Node::Empty, + // something went wrong. + _ => panic!("Rlp is not valid.") + } + } + + /// Encode the node into RLP. + /// + /// Will always return the direct node RLP even if it's 32 or more bytes. To get the + /// RLP which would be valid for using in another node, use `encoded_and_added()`. + pub fn encoded(&self) -> Bytes { + match *self { + Node::Leaf(ref slice, ref value) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(true)); + stream.append(value); + stream.out() + }, + Node::Extension(ref slice, ref raw_rlp) => { + let mut stream = RlpStream::new_list(2); + stream.append(&slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + stream.out() + }, + Node::Branch(ref nodes, ref value) => { + let mut stream = RlpStream::new_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(n) => { stream.append(&n); }, + None => { stream.append_empty_data(); }, + } + stream.out() + }, + Node::Empty => { + let mut stream = RlpStream::new(); + stream.append_empty_data(); + stream.out() + } + } + } + + /// Encode the node, adding it to `journal` if necessary and return the RLP valid for + /// insertion into a parent node. + pub fn encoded_and_added(&self, journal: &mut Journal) -> Bytes { + let mut stream = RlpStream::new(); + match *self { + Node::Leaf(ref slice, ref value) => { + stream.append_list(2); + stream.append(&slice.encoded(true)); + stream.append(value); + }, + Node::Extension(ref slice, ref raw_rlp) => { + stream.append_list(2); + stream.append(&slice.encoded(false)); + stream.append_raw(raw_rlp, 1); + }, + Node::Branch(ref nodes, ref value) => { + stream.append_list(17); + for i in 0..16 { + stream.append_raw(nodes[i], 1); + } + match *value { + Some(n) => { stream.append(&n); }, + None => { stream.append_empty_data(); }, + } + }, + Node::Empty => { + stream.append_empty_data(); + } + } + let node = stream.out(); + match node.len() { + 0 ... 31 => node, + _ => { + let mut stream = RlpStream::new(); + journal.new_node(node, &mut stream); + stream.out() + } + } + } +} diff --git a/src/trie/standardmap.rs b/src/trie/standardmap.rs new file mode 100644 index 000000000..b1ca03d22 --- /dev/null +++ b/src/trie/standardmap.rs @@ -0,0 +1,75 @@ +//! Key-value datastore with a modified Merkle tree. +extern crate rand; + +use bytes::*; +use sha3::*; +use hash::*; + +/// Alphabet to use when creating words for insertion into tries. +pub enum Alphabet { + All, + Low, + Mid, + Custom(Bytes), +} + +/// Standard test map for profiling tries. +pub struct StandardMap { + alphabet: Alphabet, + min_key: usize, + journal_key: usize, + count: usize, +} + +impl StandardMap { + /// Get a bunch of random bytes, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. + /// `seed` is mutated pseudoramdonly and used. + fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + journal_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); + seed.bytes()[0..r].to_vec() + } + + /// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used. + fn random_value(seed: &mut H256) -> Bytes { + *seed = seed.sha3(); + match seed.bytes()[0] % 2 { + 1 => vec![seed.bytes()[31];1], + _ => seed.bytes().to_vec(), + } + } + + /// Get a random word of, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. + /// Each byte is an item from `alphabet`. `seed` is mutated pseudoramdonly and used. + fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { + assert!(min_count + journal_count <= 32); + *seed = seed.sha3(); + let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); + let mut ret: Vec = Vec::with_capacity(r); + for i in 0..r { + ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); + } + ret + } + + /// Create the standard map (set of keys and values) for the object's fields. + pub fn make(&self) -> Vec<(Bytes, Bytes)> { + let low = b"abcdef"; + let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; + + let mut d: Vec<(Bytes, Bytes)> = Vec::new(); + let mut seed = H256::new(); + for _ in 0..self.count { + let k = match self.alphabet { + Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed), + Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed), + Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed), + Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed), + }; + let v = Self::random_value(&mut seed); + d.push((k, v)) + } + d + } +} diff --git a/src/trie/triedb.rs b/src/trie/triedb.rs new file mode 100644 index 000000000..5ee30249b --- /dev/null +++ b/src/trie/triedb.rs @@ -0,0 +1,225 @@ +use std::fmt; +use hashdb::*; +use hash::*; +use nibbleslice::*; +use bytes::*; +use rlp::*; +use std::collections::HashMap; +use super::trietraits::*; +use super::node::*; + +/// A `Trie` implementation using a generic `HashDB` backing database. +/// +/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` +/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get +/// which items in the backing database do not belong to this trie. If this is the only trie in the +/// backing database, then `db_items_remaining()` should be empty. +/// +/// # Example +/// ``` +/// extern crate ethcore_util as util; +/// use util::trie::*; +/// use util::hashdb::*; +/// use util::memorydb::*; +/// use util::hash::*; +/// use util::rlp::*; +/// +/// fn main() { +/// let mut memdb = MemoryDB::new(); +/// let mut root = H256::new(); +/// let mut t = TrieDBMut::new(&mut memdb, &mut root); +/// assert!(t.is_empty()); +/// assert_eq!(*t.root(), SHA3_NULL_RLP); +/// t.insert(b"foo", b"bar"); +/// assert!(t.contains(b"foo")); +/// assert_eq!(t.get(b"foo").unwrap(), b"bar"); +/// assert!(t.db_items_remaining().is_empty()); +/// t.remove(b"foo"); +/// assert!(!t.contains(b"foo")); +/// assert!(t.db_items_remaining().is_empty()); +/// } +/// ``` +pub struct TrieDB<'db> { + db: &'db HashDB, + root: &'db H256, + pub hash_count: usize, +} + +impl<'db> TrieDB<'db> { + /// Create a new trie with the backing database `db` and `root` + /// Panics, if `root` does not exist + pub fn new(db: &'db HashDB, root: &'db H256) -> Self { + assert!(db.exists(root)); + TrieDB { + db: db, + root: root, + hash_count: 0 + } + } + + /// Get the backing database. + pub fn db(&'db self) -> &'db HashDB { + self.db + } + + /// Determine all the keys in the backing database that belong to the trie. + pub fn keys(&self) -> Vec { + let mut ret: Vec = Vec::new(); + ret.push(self.root.clone()); + self.accumulate_keys(self.root_node(), &mut ret); + ret + } + + /// Convert a vector of hashes to a hashmap of hash to occurances. + pub fn to_map(hashes: Vec) -> HashMap { + let mut r: HashMap = HashMap::new(); + for h in hashes.into_iter() { + let c = *r.get(&h).unwrap_or(&0); + r.insert(h, c + 1); + } + r + } + + /// Determine occurances of items in the backing database which are not related to this + /// trie. + pub fn db_items_remaining(&self) -> HashMap { + let mut ret = self.db.keys(); + for (k, v) in Self::to_map(self.keys()).into_iter() { + let keycount = *ret.get(&k).unwrap_or(&0); + match keycount == v as i32 { + true => ret.remove(&k), + _ => ret.insert(k, keycount - v as i32), + }; + } + ret + } + + /// Recursion helper for `keys`. + fn accumulate_keys(&self, node: Node, acc: &mut Vec) { + let mut handle_payload = |payload| { + let p = Rlp::new(payload); + if p.is_data() && p.size() == 32 { + acc.push(p.as_val()); + } + + self.accumulate_keys(self.get_node(payload), acc); + }; + + match node { + Node::Extension(_, payload) => handle_payload(payload), + Node::Branch(payloads, _) => for payload in payloads.iter() { handle_payload(payload) }, + _ => {}, + } + } + + /// Get the root node's RLP. + fn root_node(&self) -> Node { + Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) + } + + /// Get the root node as a `Node`. + fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { + Node::decoded(self.get_raw_or_lookup(node)) + } + + /// Indentation helper for `formal_all`. + fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { + for _ in 0..size { + try!(write!(f, " ")); + } + Ok(()) + } + + /// Recursion helper for implementation of formatting trait. + fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { + match node { + Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())), + Node::Extension(ref slice, ref item) => { + try!(write!(f, "'{:?} ", slice)); + try!(self.fmt_all(self.get_node(item), f, deepness)); + }, + Node::Branch(ref nodes, ref value) => { + try!(writeln!(f, "")); + match value { + &Some(v) => { + try!(self.fmt_indent(f, deepness + 1)); + try!(writeln!(f, "=: {:?}", v.pretty())) + }, + &None => {} + } + for i in 0..16 { + match self.get_node(nodes[i]) { + Node::Empty => {}, + n => { + try!(self.fmt_indent(f, deepness + 1)); + try!(write!(f, "'{:x} ", i)); + try!(self.fmt_all(n, f, deepness + 1)); + } + } + } + }, + // empty + Node::Empty => { + try!(writeln!(f, "")); + } + }; + Ok(()) + } + + /// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists. + fn do_lookup<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + self.get_from_node(&root_rlp, key) + } + + /// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no + /// value exists for the key. + /// + /// Note: Not a public API; use Trie trait functions. + fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { + match Node::decoded(node) { + Node::Leaf(ref slice, ref value) if key == slice => Some(value), + Node::Extension(ref slice, ref item) if key.starts_with(slice) => { + self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len())) + }, + Node::Branch(ref nodes, value) => match key.is_empty() { + true => value, + false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1)) + }, + _ => None + } + } + + /// Given some node-describing data `node`, return the actual node RLP. + /// This could be a simple identity operation in the case that the node is sufficiently small, but + /// may require a database lookup. + fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { + // check if its sha3 + len + let r = Rlp::new(node); + match r.is_data() && r.size() == 32 { + true => self.db.lookup(&r.as_val::()).expect("Not found!"), + false => node + } + } +} + +impl<'db> Trie for TrieDB<'db> { + fn root(&self) -> &H256 { &self.root } + + fn contains(&self, key: &[u8]) -> bool { + self.get(key).is_some() + } + + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.do_lookup(&NibbleSlice::new(key)) + } +} + +impl<'db> fmt::Debug for TrieDB<'db> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(writeln!(f, "c={:?} [", self.hash_count)); + let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); + try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); + writeln!(f, "]") + } +} diff --git a/src/trie.rs b/src/trie/triedbmut.rs similarity index 70% rename from src/trie.rs rename to src/trie/triedbmut.rs index 6c0f6dc78..fda48ef23 100644 --- a/src/trie.rs +++ b/src/trie/triedbmut.rs @@ -1,312 +1,13 @@ -//! Key-value datastore with a modified Merkle tree. -extern crate rand; - use std::fmt; -use sha3::*; use hashdb::*; use hash::*; use nibbleslice::*; use bytes::*; use rlp::*; use std::collections::HashMap; - -//use log::*; - -pub const NULL_RLP: [u8; 1] = [0x80; 1]; -pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); - -/// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait Trie { - /// Return the root of the trie. - fn root(&self) -> &H256; - - /// Is the trie empty? - fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } - - /// Does the trie contain a given key? - fn contains(&self, key: &[u8]) -> bool; - - /// What is the value of the given key in this trie? - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; -} - -/// A key-value datastore implemented as a database-backed modified Merkle tree. -pub trait TrieMut: Trie { - /// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing - /// `key` from the trie. - fn insert(&mut self, key: &[u8], value: &[u8]); - - /// Remove a `key` from the trie. Equivalent to making it equal to the empty - /// value. - fn remove(&mut self, key: &[u8]); -} - -/// Alphabet to use when creating words for insertion into tries. -pub enum Alphabet { - All, - Low, - Mid, - Custom(Bytes), -} - -/// Standard test map for profiling tries. -pub struct StandardMap { - alphabet: Alphabet, - min_key: usize, - journal_key: usize, - count: usize, -} - -impl StandardMap { - /// Get a bunch of random bytes, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. - /// `seed` is mutated pseudoramdonly and used. - fn random_bytes(min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { - assert!(min_count + journal_count <= 32); - *seed = seed.sha3(); - let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); - seed.bytes()[0..r].to_vec() - } - - /// Get a random value. Equal chance of being 1 byte as of 32. `seed` is mutated pseudoramdonly and used. - fn random_value(seed: &mut H256) -> Bytes { - *seed = seed.sha3(); - match seed.bytes()[0] % 2 { - 1 => vec![seed.bytes()[31];1], - _ => seed.bytes().to_vec(), - } - } - - /// Get a random word of, at least `min_count` bytes, at most `min_count` + `journal_count` bytes. - /// Each byte is an item from `alphabet`. `seed` is mutated pseudoramdonly and used. - fn random_word(alphabet: &[u8], min_count: usize, journal_count: usize, seed: &mut H256) -> Vec { - assert!(min_count + journal_count <= 32); - *seed = seed.sha3(); - let r = min_count + (seed.bytes()[31] as usize % (journal_count + 1)); - let mut ret: Vec = Vec::with_capacity(r); - for i in 0..r { - ret.push(alphabet[seed.bytes()[i] as usize % alphabet.len()]); - } - ret - } - - /// Create the standard map (set of keys and values) for the object's fields. - pub fn make(&self) -> Vec<(Bytes, Bytes)> { - let low = b"abcdef"; - let mid = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; - - let mut d: Vec<(Bytes, Bytes)> = Vec::new(); - let mut seed = H256::new(); - for _ in 0..self.count { - let k = match self.alphabet { - Alphabet::All => Self::random_bytes(self.min_key, self.journal_key, &mut seed), - Alphabet::Low => Self::random_word(low, self.min_key, self.journal_key, &mut seed), - Alphabet::Mid => Self::random_word(mid, self.min_key, self.journal_key, &mut seed), - Alphabet::Custom(ref a) => Self::random_word(&a, self.min_key, self.journal_key, &mut seed), - }; - let v = Self::random_value(&mut seed); - d.push((k, v)) - } - d - } -} - -/// Type of node in the trie and essential information thereof. -#[derive(Eq, PartialEq, Debug)] -pub enum Node<'a> { - Empty, - Leaf(NibbleSlice<'a>, &'a[u8]), - Extension(NibbleSlice<'a>, &'a[u8]), - Branch([&'a[u8]; 16], Option<&'a [u8]>) -} - -/// Type of operation for the backing database - either a new node or a node deletion. -#[derive(Debug)] -enum Operation { - New(H256, Bytes), - Delete(H256), -} - -/// A journal of operations on the backing database. -#[derive(Debug)] -struct Journal (Vec); - -impl Journal { - /// Create a new, empty, object. - fn new() -> Journal { Journal(vec![]) } - - /// Given the RLP that encodes a node, append a reference to that node `out` and leave `journal` - /// such that the reference is valid, once applied. - fn new_node(&mut self, rlp: Bytes, out: &mut RlpStream) { - if rlp.len() >= 32 { - let rlp_sha3 = rlp.sha3(); - - trace!("new_node: reference node {:?} => {:?}", rlp_sha3, rlp.pretty()); - out.append(&rlp_sha3); - self.0.push(Operation::New(rlp_sha3, rlp)); - } - else { - trace!("new_node: inline node {:?}", rlp.pretty()); - out.append_raw(&rlp, 1); - } - } - - /// Given the RLP that encodes a now-unused node, leave `journal` in such a state that it is noted. - fn delete_node_sha3(&mut self, old_sha3: H256) { - trace!("delete_node: {:?}", old_sha3); - self.0.push(Operation::Delete(old_sha3)); - } - - /// Register an RLP-encoded node for deletion (given a slice), if it needs to be deleted. - fn delete_node(&mut self, old: &[u8]) { - let r = Rlp::new(old); - if r.is_data() && r.size() == 32 { - self.delete_node_sha3(r.as_val()); - } - } -} - -impl <'a>Node<'a> { - /// Decode the `node_rlp` and return the Node. - fn decoded(node_rlp: &'a [u8]) -> Node<'a> { - let r = Rlp::new(node_rlp); - match r.prototype() { - // either leaf or extension - decode first item with NibbleSlice::??? - // and use is_leaf return to figure out which. - // if leaf, second item is a value (is_data()) - // if extension, second item is a node (either SHA3 to be looked up and - // fed back into this function or inline RLP which can be fed back into this function). - Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { - (slice, true) => Node::Leaf(slice, r.at(1).data()), - (slice, false) => Node::Extension(slice, r.at(1).raw()), - }, - // branch - first 16 are nodes, 17th is a value (or empty). - Prototype::List(17) => { - let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; - for i in 0..16 { - nodes[i] = r.at(i).raw(); - } - Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) - }, - // an empty branch index. - Prototype::Data(0) => Node::Empty, - // something went wrong. - _ => panic!("Rlp is not valid.") - } - } - - /// Encode the node into RLP. - /// - /// Will always return the direct node RLP even if it's 32 or more bytes. To get the - /// RLP which would be valid for using in another node, use `encoded_and_added()`. - fn encoded(&self) -> Bytes { - match *self { - Node::Leaf(ref slice, ref value) => { - let mut stream = RlpStream::new_list(2); - stream.append(&slice.encoded(true)); - stream.append(value); - stream.out() - }, - Node::Extension(ref slice, ref raw_rlp) => { - let mut stream = RlpStream::new_list(2); - stream.append(&slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - stream.out() - }, - Node::Branch(ref nodes, ref value) => { - let mut stream = RlpStream::new_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(n) => { stream.append(&n); }, - None => { stream.append_empty_data(); }, - } - stream.out() - }, - Node::Empty => { - let mut stream = RlpStream::new(); - stream.append_empty_data(); - stream.out() - } - } - } - - /// Encode the node, adding it to `journal` if necessary and return the RLP valid for - /// insertion into a parent node. - fn encoded_and_added(&self, journal: &mut Journal) -> Bytes { - let mut stream = RlpStream::new(); - match *self { - Node::Leaf(ref slice, ref value) => { - stream.append_list(2); - stream.append(&slice.encoded(true)); - stream.append(value); - }, - Node::Extension(ref slice, ref raw_rlp) => { - stream.append_list(2); - stream.append(&slice.encoded(false)); - stream.append_raw(raw_rlp, 1); - }, - Node::Branch(ref nodes, ref value) => { - stream.append_list(17); - for i in 0..16 { - stream.append_raw(nodes[i], 1); - } - match *value { - Some(n) => { stream.append(&n); }, - None => { stream.append_empty_data(); }, - } - }, - Node::Empty => { - stream.append_empty_data(); - } - } - let node = stream.out(); - match node.len() { - 0 ... 31 => node, - _ => { - let mut stream = RlpStream::new(); - journal.new_node(node, &mut stream); - stream.out() - } - } - } -} - -/// A `Trie` implementation using a generic `HashDB` backing database. -/// -/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` -/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get -/// which items in the backing database do not belong to this trie. If this is the only trie in the -/// backing database, then `db_items_remaining()` should be empty. -/// -/// # Example -/// ``` -/// extern crate ethcore_util as util; -/// use util::trie::*; -/// use util::hashdb::*; -/// use util::memorydb::*; -/// use util::hash::*; -/// -/// fn main() { -/// let mut memdb = MemoryDB::new(); -/// let mut root = H256::new(); -/// let mut t = TrieDBMut::new(&mut memdb, &mut root); -/// assert!(t.is_empty()); -/// assert_eq!(*t.root(), SHA3_NULL_RLP); -/// t.insert(b"foo", b"bar"); -/// assert!(t.contains(b"foo")); -/// assert_eq!(t.get(b"foo").unwrap(), b"bar"); -/// assert!(t.db_items_remaining().is_empty()); -/// t.remove(b"foo"); -/// assert!(!t.contains(b"foo")); -/// assert!(t.db_items_remaining().is_empty()); -/// } -/// ``` -pub struct TrieDB<'db> { - db: &'db HashDB, - root: &'db H256, - pub hash_count: usize, -} +use super::node::*; +use super::journal::*; +use super::trietraits::*; pub struct TrieDBMut<'db> { db: &'db mut HashDB, @@ -320,185 +21,6 @@ enum MaybeChanged<'a> { Changed(Bytes), } -impl<'db> TrieDB<'db> { - /// Create a new trie with the backing database `db` and `root` - /// Panics, if `root` does not exist - pub fn new(db: &'db HashDB, root: &'db H256) -> Self { - assert!(db.exists(root)); - TrieDB { - db: db, - root: root, - hash_count: 0 - } - } - - /// Get the backing database. - pub fn db(&'db self) -> &'db HashDB { - self.db - } - - /// Determine all the keys in the backing database that belong to the trie. - pub fn keys(&self) -> Vec { - let mut ret: Vec = Vec::new(); - ret.push(self.root.clone()); - self.accumulate_keys(self.root_node(), &mut ret); - ret - } - - /// Convert a vector of hashes to a hashmap of hash to occurances. - pub fn to_map(hashes: Vec) -> HashMap { - let mut r: HashMap = HashMap::new(); - for h in hashes.into_iter() { - let c = *r.get(&h).unwrap_or(&0); - r.insert(h, c + 1); - } - r - } - - /// Determine occurances of items in the backing database which are not related to this - /// trie. - pub fn db_items_remaining(&self) -> HashMap { - let mut ret = self.db.keys(); - for (k, v) in Self::to_map(self.keys()).into_iter() { - let keycount = *ret.get(&k).unwrap_or(&0); - match keycount == v as i32 { - true => ret.remove(&k), - _ => ret.insert(k, keycount - v as i32), - }; - } - ret - } - - /// Recursion helper for `keys`. - fn accumulate_keys(&self, node: Node, acc: &mut Vec) { - let mut handle_payload = |payload| { - let p = Rlp::new(payload); - if p.is_data() && p.size() == 32 { - acc.push(p.as_val()); - } - - self.accumulate_keys(self.get_node(payload), acc); - }; - - match node { - Node::Extension(_, payload) => handle_payload(payload), - Node::Branch(payloads, _) => for payload in payloads.iter() { handle_payload(payload) }, - _ => {}, - } - } - - /// Get the root node's RLP. - fn root_node(&self) -> Node { - Node::decoded(self.db.lookup(&self.root).expect("Trie root not found!")) - } - - /// Get the root node as a `Node`. - fn get_node<'a>(&'a self, node: &'a [u8]) -> Node { - Node::decoded(self.get_raw_or_lookup(node)) - } - - /// Indentation helper for `formal_all`. - fn fmt_indent(&self, f: &mut fmt::Formatter, size: usize) -> fmt::Result { - for _ in 0..size { - try!(write!(f, " ")); - } - Ok(()) - } - - /// Recursion helper for implementation of formatting trait. - fn fmt_all(&self, node: Node, f: &mut fmt::Formatter, deepness: usize) -> fmt::Result { - match node { - Node::Leaf(slice, value) => try!(writeln!(f, "'{:?}: {:?}.", slice, value.pretty())), - Node::Extension(ref slice, ref item) => { - try!(write!(f, "'{:?} ", slice)); - try!(self.fmt_all(self.get_node(item), f, deepness)); - }, - Node::Branch(ref nodes, ref value) => { - try!(writeln!(f, "")); - match value { - &Some(v) => { - try!(self.fmt_indent(f, deepness + 1)); - try!(writeln!(f, "=: {:?}", v.pretty())) - }, - &None => {} - } - for i in 0..16 { - match self.get_node(nodes[i]) { - Node::Empty => {}, - n => { - try!(self.fmt_indent(f, deepness + 1)); - try!(write!(f, "'{:x} ", i)); - try!(self.fmt_all(n, f, deepness + 1)); - } - } - } - }, - // empty - Node::Empty => { - try!(writeln!(f, "")); - } - }; - Ok(()) - } - - /// Return optional data for a key given as a `NibbleSlice`. Returns `None` if no data exists. - fn do_lookup<'a, 'key>(&'a self, key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - self.get_from_node(&root_rlp, key) - } - - /// Recursible function to retrieve the value given a `node` and a partial `key`. `None` if no - /// value exists for the key. - /// - /// Note: Not a public API; use Trie trait functions. - fn get_from_node<'a, 'key>(&'a self, node: &'a [u8], key: &NibbleSlice<'key>) -> Option<&'a [u8]> where 'a: 'key { - match Node::decoded(node) { - Node::Leaf(ref slice, ref value) if key == slice => Some(value), - Node::Extension(ref slice, ref item) if key.starts_with(slice) => { - self.get_from_node(self.get_raw_or_lookup(item), &key.mid(slice.len())) - }, - Node::Branch(ref nodes, value) => match key.is_empty() { - true => value, - false => self.get_from_node(self.get_raw_or_lookup(nodes[key.at(0) as usize]), &key.mid(1)) - }, - _ => None - } - } - - /// Given some node-describing data `node`, return the actual node RLP. - /// This could be a simple identity operation in the case that the node is sufficiently small, but - /// may require a database lookup. - fn get_raw_or_lookup<'a>(&'a self, node: &'a [u8]) -> &'a [u8] { - // check if its sha3 + len - let r = Rlp::new(node); - match r.is_data() && r.size() == 32 { - true => self.db.lookup(&r.as_val::()).expect("Not found!"), - false => node - } - } -} - -impl<'db> Trie for TrieDB<'db> { - fn root(&self) -> &H256 { &self.root } - - fn contains(&self, key: &[u8]) -> bool { - self.get(key).is_some() - } - - fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { - self.do_lookup(&NibbleSlice::new(key)) - } -} - -impl<'db> fmt::Debug for TrieDB<'db> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - try!(writeln!(f, "c={:?} [", self.hash_count)); - let root_rlp = self.db.lookup(&self.root).expect("Trie root not found!"); - try!(self.fmt_all(Node::decoded(root_rlp), f, 0)); - writeln!(f, "]") - } -} - impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. @@ -574,20 +96,7 @@ impl<'db> TrieDBMut<'db> { /// Apply the items in `journal` into the backing database. fn apply(&mut self, journal: Journal) { - trace!("applying {:?} changes", journal.0.len()); - for d in journal.0.into_iter() { - match d { - Operation::Delete(h) => { - trace!("TrieDBMut::apply --- {:?}", &h); - self.db.kill(&h); - }, - Operation::New(h, d) => { - trace!("TrieDBMut::apply +++ {:?} -> {:?}", &h, d.pretty()); - self.db.emplace(h, d); - self.hash_count += 1; - } - } - } + self.hash_count += journal.apply(self.db).inserts; } /// Recursion helper for `keys`. @@ -1124,11 +633,13 @@ mod tests { use memorydb::*; use super::*; use nibbleslice::*; - use rlp; + use rlp::*; use env_logger; use rand::random; use std::collections::HashSet; use bytes::{ToPretty,Bytes}; + use super::super::node::*; + use super::super::trietraits::*; fn random_key(alphabet: &[u8], min_count: usize, journal_count: usize) -> Vec { let mut ret: Vec = Vec::new(); @@ -1141,11 +652,11 @@ mod tests { fn random_value_indexed(j: usize) -> Bytes { match random::() % 2 { - 0 => rlp::encode(&j), + 0 => encode(&j), _ => { let mut h = H256::new(); h.mut_bytes()[31] = j as u8; - rlp::encode(&h) + encode(&h) }, } } @@ -1408,7 +919,7 @@ mod tests { fn test_node_extension() { let k = vec![0x00u8, 0x01, 0x23, 0x45]; // in extension, value must be valid rlp - let v = rlp::encode(&"cat"); + let v = encode(&"cat"); let (slice, is_leaf) = NibbleSlice::from_encoded(&k); assert_eq!(is_leaf, false); let ex = Node::Extension(slice, &v); @@ -1429,7 +940,7 @@ mod tests { #[test] fn test_node_branch() { - let k = rlp::encode(&"cat"); + let k = encode(&"cat"); let mut nodes: [&[u8]; 16] = unsafe { ::std::mem::uninitialized() }; for i in 0..16 { nodes[i] = &k; } let v: Vec = From::from("dog"); @@ -1491,7 +1002,7 @@ mod tests { let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; for j in 0..4u32 { let key = random_key(alphabet, 5, 1); - x.push((key, rlp::encode(&j))); + x.push((key, encode(&j))); } let real = trie_root(x.clone()); let mut memdb = MemoryDB::new(); diff --git a/src/trie/trietraits.rs b/src/trie/trietraits.rs new file mode 100644 index 000000000..5b069fdd5 --- /dev/null +++ b/src/trie/trietraits.rs @@ -0,0 +1,29 @@ +use hash::H256; +use rlp::SHA3_NULL_RLP; + +/// A key-value datastore implemented as a database-backed modified Merkle tree. +pub trait Trie { + /// Return the root of the trie. + fn root(&self) -> &H256; + + /// Is the trie empty? + fn is_empty(&self) -> bool { *self.root() == SHA3_NULL_RLP } + + /// Does the trie contain a given key? + fn contains(&self, key: &[u8]) -> bool; + + /// What is the value of the given key in this trie? + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key; +} + +/// A key-value datastore implemented as a database-backed modified Merkle tree. +pub trait TrieMut: Trie { + /// Insert a `key`/`value` pair into the trie. An `empty` value is equivalent to removing + /// `key` from the trie. + fn insert(&mut self, key: &[u8], value: &[u8]); + + /// Remove a `key` from the trie. Equivalent to making it equal to the empty + /// value. + fn remove(&mut self, key: &[u8]); +} + From 0addfc186d127493eef39e91c439b9f392107755 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 19 Dec 2015 12:53:05 +0000 Subject: [PATCH 271/379] Add tests for triedb, optimise tests for triedbmut. --- src/trie/triedb.rs | 9 ++------- src/trie/triedbmut.rs | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/trie/triedb.rs b/src/trie/triedb.rs index 5ee30249b..ef07fd47d 100644 --- a/src/trie/triedb.rs +++ b/src/trie/triedb.rs @@ -27,16 +27,11 @@ use super::node::*; /// fn main() { /// let mut memdb = MemoryDB::new(); /// let mut root = H256::new(); -/// let mut t = TrieDBMut::new(&mut memdb, &mut root); -/// assert!(t.is_empty()); -/// assert_eq!(*t.root(), SHA3_NULL_RLP); -/// t.insert(b"foo", b"bar"); +/// TrieDBMut::new(&mut memdb, &mut root).insert(b"foo", b"bar"); +/// let t = TrieDB::new(&memdb, &root); /// assert!(t.contains(b"foo")); /// assert_eq!(t.get(b"foo").unwrap(), b"bar"); /// assert!(t.db_items_remaining().is_empty()); -/// t.remove(b"foo"); -/// assert!(!t.contains(b"foo")); -/// assert!(t.db_items_remaining().is_empty()); /// } /// ``` pub struct TrieDB<'db> { diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index fda48ef23..004912a4c 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -21,6 +21,37 @@ enum MaybeChanged<'a> { Changed(Bytes), } +/// A `Trie` implementation using a generic `HashDB` backing database. +/// +/// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` +/// to get the keys belonging to the trie in the backing database, and `db_items_remaining()` to get +/// which items in the backing database do not belong to this trie. If this is the only trie in the +/// backing database, then `db_items_remaining()` should be empty. +/// +/// # Example +/// ``` +/// extern crate ethcore_util as util; +/// use util::trie::*; +/// use util::hashdb::*; +/// use util::memorydb::*; +/// use util::hash::*; +/// use util::rlp::*; +/// +/// fn main() { +/// let mut memdb = MemoryDB::new(); +/// let mut root = H256::new(); +/// let mut t = TrieDBMut::new(&mut memdb, &mut root); +/// assert!(t.is_empty()); +/// assert_eq!(*t.root(), SHA3_NULL_RLP); +/// t.insert(b"foo", b"bar"); +/// assert!(t.contains(b"foo")); +/// assert_eq!(t.get(b"foo").unwrap(), b"bar"); +/// assert!(t.db_items_remaining().is_empty()); +/// t.remove(b"foo"); +/// assert!(!t.contains(b"foo")); +/// assert!(t.db_items_remaining().is_empty()); +/// } +/// ``` impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. @@ -712,7 +743,7 @@ mod tests { let mut x: Vec<(Vec, Vec)> = Vec::new(); let mut got: HashSet> = HashSet::new(); let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; - for j in 0..1000usize { + for j in 0..100usize { let key = random_key(alphabet, 5, 0); if !got.contains(&key) { x.push((key.clone(), random_value_indexed(j))); @@ -997,7 +1028,7 @@ mod tests { #[test] fn stress() { - for _ in 0..500 { + for _ in 0..50 { let mut x: Vec<(Vec, Vec)> = Vec::new(); let alphabet = b"@QWERTYUIOPASDFGHJKLZXCVBNM[/]^_"; for j in 0..4u32 { From 72a24ad023a318909c32251b972067edf8bdc5c6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 20 Dec 2015 12:28:13 +0000 Subject: [PATCH 272/379] SemanticVersion struct. Additional documentation. --- src/lib.rs | 1 + src/semantic_version.rs | 28 ++++++++++++++++++++++++++++ src/trie/journal.rs | 4 ++++ 3 files changed, 33 insertions(+) create mode 100644 src/semantic_version.rs diff --git a/src/lib.rs b/src/lib.rs index 9120f0977..4859a8580 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,5 +66,6 @@ pub mod trie; pub mod nibbleslice; pub mod heapsizeof; pub mod squeeze; +pub mod semantic_version; //pub mod network; diff --git a/src/semantic_version.rs b/src/semantic_version.rs new file mode 100644 index 000000000..026902263 --- /dev/null +++ b/src/semantic_version.rs @@ -0,0 +1,28 @@ +/// A version value with strict meaning. Use `to_u32` to convert to a simple integer. +/// +/// # Example +/// ``` +/// extern crate ethcore; +/// use ethcore::engine::*; +/// fn main() { +/// assert_eq!(SemanticVersion::new(1, 2, 3).as_u32(), 0x010203); +/// } +/// ``` +pub struct SemanticVersion { + /// Major version - API/feature removals & breaking changes. + pub major: u8, + /// Minor version - API/feature additions. + pub minor: u8, + /// Tiny version - bug fixes. + pub tiny: u8, +} + +impl SemanticVersion { + /// Create a new object. + pub fn new(major: u8, minor: u8, tiny: u8) -> SemanticVersion { SemanticVersion{major: major, minor: minor, tiny: tiny} } + + /// Convert to a `u32` representation. + pub fn as_u32(&self) -> u32 { ((self.major as u32) << 16) + ((self.minor as u32) << 8) + self.tiny as u32 } +} + +// TODO: implement Eq, Comparison and Debug/Display for SemanticVersion. diff --git a/src/trie/journal.rs b/src/trie/journal.rs index 77efc991c..948f07a06 100644 --- a/src/trie/journal.rs +++ b/src/trie/journal.rs @@ -11,8 +11,11 @@ enum Operation { Delete(H256), } +/// How many insertions and removals were done in an `apply` operation. pub struct Score { + /// Number of insertions. pub inserts: usize, + /// Number of removals. pub removes: usize, } @@ -54,6 +57,7 @@ impl Journal { } } + /// Apply this journal to the HashDB `db` and return the number of insertions and removals done. pub fn apply(self, db: &mut HashDB) -> Score { trace!("applying {:?} changes", self.0.len()); let mut ret = Score{inserts: 0, removes: 0}; From be2b041ee4d0166aa96222ea0c6602dc516c46e1 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 21 Dec 2015 01:57:34 +0100 Subject: [PATCH 273/379] fixed semantic version tests --- src/semantic_version.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/semantic_version.rs b/src/semantic_version.rs index 026902263..92f6ea376 100644 --- a/src/semantic_version.rs +++ b/src/semantic_version.rs @@ -2,8 +2,9 @@ /// /// # Example /// ``` -/// extern crate ethcore; -/// use ethcore::engine::*; +/// extern crate ethcore_util as util; +/// use util::semantic_version::*; +/// /// fn main() { /// assert_eq!(SemanticVersion::new(1, 2, 3).as_u32(), 0x010203); /// } From 9220e0c49f7ae44c1118abee7b81e75697fcb067 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 21 Dec 2015 02:30:38 +0100 Subject: [PATCH 274/379] missing Error case --- src/error.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index 71514d045..e41bcd178 100644 --- a/src/error.rs +++ b/src/error.rs @@ -13,6 +13,7 @@ pub enum EthcoreError { FromHex(FromHexError), BaseData(BaseDataError), BadSize, + UnknownName } impl From for EthcoreError { @@ -39,4 +40,4 @@ macro_rules! assimilate { ) } assimilate!(FromHex); -assimilate!(BaseData);*/ \ No newline at end of file +assimilate!(BaseData);*/ From f34b22c65b236e9b253fb3a117177d0ebf0d987e Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 22 Dec 2015 22:23:01 +0100 Subject: [PATCH 275/379] cache item count in Rlp --- src/rlp/rlpstream.rs | 8 ++++---- src/rlp/untrusted_rlp.rs | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 43ae83a9a..9868d91e9 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -26,7 +26,7 @@ pub struct RlpStream { } impl Stream for RlpStream { - fn new() -> Self { + fn new() -> Self { RlpStream { unfinished_lists: ElasticArray16::new(), encoder: BasicEncoder::new(), @@ -57,7 +57,7 @@ impl Stream for RlpStream { self.encoder.bytes.push(0xc0u8); self.note_appended(1); }, - _ => { + _ => { let position = self.encoder.bytes.len(); self.unfinished_lists.push(ListInfo::new(position, len)); }, @@ -66,7 +66,7 @@ impl Stream for RlpStream { // return chainable self self } - + fn append_empty_data<'a>(&'a mut self) -> &'a mut RlpStream { // self push raw item self.encoder.bytes.push(0x80); @@ -80,7 +80,7 @@ impl Stream for RlpStream { fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut RlpStream { // push raw items - self.encoder.bytes.append_slice(bytes); + self.encoder.bytes.append_slice(bytes); // try to finish and prepend the length self.note_appended(item_count); diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index a8cecf09f..d88563c30 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -41,22 +41,24 @@ impl PayloadInfo { } /// Data-oriented view onto rlp-slice. -/// +/// /// This is immutable structere. No operations change it. -/// +/// /// Should be used in places where, error handling is required, /// eg. on input #[derive(Debug)] pub struct UntrustedRlp<'a> { bytes: &'a [u8], - cache: Cell, + offset_cache: Cell, + count_cache: Cell>, } impl<'a> Clone for UntrustedRlp<'a> { fn clone(&self) -> UntrustedRlp<'a> { UntrustedRlp { bytes: self.bytes, - cache: Cell::new(OffsetCache::new(usize::max_value(), 0)) + offset_cache: self.offset_cache.clone(), + count_cache: self.count_cache.clone(), } } } @@ -72,10 +74,11 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { fn new(bytes: &'a [u8]) -> UntrustedRlp<'a> { UntrustedRlp { bytes: bytes, - cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), + offset_cache: Cell::new(OffsetCache::new(usize::max_value(), 0)), + count_cache: Cell::new(None) } } - + fn raw(&'view self) -> &'a [u8] { self.bytes } @@ -102,7 +105,14 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { fn item_count(&self) -> usize { match self.is_list() { - true => self.iter().count(), + true => match self.count_cache.get() { + Some(c) => c, + None => { + let c = self.iter().count(); + self.count_cache.set(Some(c)); + c + } + }, false => 0 } } @@ -122,7 +132,7 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { // move to cached position if it's index is less or equal to // current search index, otherwise move to beginning of list - let c = self.cache.get(); + let c = self.offset_cache.get(); let (mut bytes, to_skip) = match c.index <= index { true => (try!(UntrustedRlp::consume(self.bytes, c.offset)), index - c.index), false => (try!(self.consume_list_prefix()), index), @@ -132,7 +142,7 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { bytes = try!(UntrustedRlp::consume_items(bytes, to_skip)); // update the cache - self.cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); + self.offset_cache.set(OffsetCache::new(index, self.bytes.len() - bytes.len())); // construct new rlp let found = try!(BasicDecoder::payload_info(bytes)); @@ -360,7 +370,7 @@ macro_rules! impl_array_decodable { if decoders.len() != $len { return Err(DecoderError::RlpIncorrectListLen); } - + for i in 0..decoders.len() { result[i] = try!(T::decode(&decoders[i])); } From 3ad262e18fd2ac2755e8abd292848d15cb1fc82c Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 22 Dec 2015 22:23:43 +0100 Subject: [PATCH 276/379] more networking --- Cargo.toml | 4 +- src/network/host.rs | 108 +++++++++++++++++++++++--------------------- src/network/mod.rs | 7 ++- 3 files changed, 64 insertions(+), 55 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 85a1f35a6..0e584d8d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,12 +11,12 @@ log = "0.3" env_logger = "0.3" rustc-serialize = "0.3" arrayvec = "0.3" -mio = "0.5.*" +mio = "0.5.0" rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" rocksdb = "0.2.1" -lazy_static = "0.1.*" +lazy_static = "0.1.0" eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } rust-crypto = "0.2.34" elastic-array = "0.4" diff --git a/src/network/host.rs b/src/network/host.rs index 6c37bb1d3..68430c688 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -23,7 +23,7 @@ const MAX_USER_TIMERS: usize = 32; const IDEAL_PEERS:u32 = 10; pub type NodeId = H512; -type TimerFun = Fn(&mut HostIo) -> bool + Send; +pub type TimerToken = usize; #[derive(Debug)] struct NetworkConfiguration { @@ -146,7 +146,7 @@ const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; const USER_TIMER: usize = LAST_CONNECTION; const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; -pub type PacketId = u32; +pub type PacketId = u8; pub type ProtocolId = &'static str; pub enum HostMessage { @@ -162,14 +162,9 @@ pub enum HostMessage { protocol: ProtocolId, data: Vec, }, - AddTimer { - handler: Box, - delay: u32, - protocol: ProtocolId, - } } -pub type PeerId = u32; +pub type PeerId = usize; #[derive(Debug, PartialEq, Eq)] pub struct CapabilityInfo { @@ -189,54 +184,67 @@ impl Encodable for CapabilityInfo { pub struct HostIo<'s> { protocol: ProtocolId, - session: Option<&'s mut Session>, + connections: &'s mut Slab, + timers: &'s mut Slab, + session: Option, event_loop: &'s mut EventLoop, - channel: &'s mut Sender } impl<'s> HostIo<'s> { - fn new(protocol: ProtocolId, session: Option<&'s mut Session>, event_loop: &'s mut EventLoop, channel: &'s mut Sender) -> HostIo<'s> { + fn new(protocol: ProtocolId, session: Option, event_loop: &'s mut EventLoop, connections: &'s mut Slab, timers: &'s mut Slab) -> HostIo<'s> { HostIo { protocol: protocol, session: session, event_loop: event_loop, - channel: channel, + connections: connections, + timers: timers, } } pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), Error> { - try!(self.channel.send(HostMessage::Send { - peer: peer, - packet_id: packet_id, - protocol: self.protocol, - data: data - })); + match self.connections.get_mut(Token(peer)) { + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { + warn!(target: "net", "Send error: {:?}", e); + }); //TODO: don't copy vector data + }, + _ => { + warn!(target: "net", "Send: Peer does not exist"); + } + } Ok(()) } - pub fn respond(&mut self, packet_id: PacketId, data: &[u8]) -> Result<(), Error> { - match self.session.as_mut() { - Some(session) => session.send_packet(self.protocol, packet_id as u8, data), + pub fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), Error> { + match self.session { + Some(session) => self.send(session.as_usize(), packet_id, data), None => { panic!("Respond: Session does not exist") } } } - fn register_timer(&mut self, ms: u32, handler: Box) -> Result<(), Error>{ - try!(self.channel.send(HostMessage::AddTimer { - delay: ms, - handler: handler, - protocol: self.protocol, - })); - Ok(()) + pub fn register_timer(&mut self, ms: u64) -> Result{ + match self.timers.insert(UserTimer { + delay: ms, + protocol: self.protocol, + }) { + Ok(token) => { + self.event_loop.timeout_ms(token, ms).expect("Error registering user timer"); + Ok(token.as_usize()) + }, + _ => { panic!("Max timers reached") } + } + } + + pub fn disable_peer(&mut self, _peer: PeerId) { + //TODO: remove capability, disconnect if no capabilities left } } struct UserTimer { - handler: Box, protocol: ProtocolId, - delay: u32, + delay: u64, } pub struct HostInfo { @@ -492,6 +500,8 @@ impl Host { fn connection_readable(&mut self, token: Token, event_loop: &mut EventLoop) { let mut kill = false; let mut create_session = false; + let mut ready_data: Vec = Vec::new(); + let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; { match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { @@ -509,9 +519,9 @@ impl Host { }) }; match sd { SessionData::Ready => { - for (p, h) in self.handlers.iter_mut() { + for (p, _) in self.handlers.iter_mut() { if s.have_capability(p) { - h.connected(&mut HostIo::new(p, Some(s), event_loop, &mut self.channel), &(token.as_usize() as u32)); + ready_data.push(p); } } }, @@ -522,7 +532,7 @@ impl Host { } => { match self.handlers.get_mut(protocol) { None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(h) => h.read(&mut HostIo::new(protocol, Some(s), event_loop, &mut self.channel), packet_id, &data[1..]) + Some(_) => packet_data = Some((protocol, packet_id, data)), } }, SessionData::None => {}, @@ -539,6 +549,15 @@ impl Host { if create_session { self.start_session(token, event_loop); } + for p in ready_data { + let mut h = self.handlers.get_mut(p).unwrap(); + h.connected(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize()); + } + if let Some((p, packet_id, data)) = packet_data { + let mut h = self.handlers.get_mut(p).unwrap(); + h.read(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize(), packet_id, &data[1..]); + } + } fn start_session(&mut self, token: Token, event_loop: &mut EventLoop) { @@ -593,9 +612,10 @@ impl Handler for Host { NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, USER_TIMER ... LAST_USER_TIMER => { - let timer = self.timers.get_mut(token).expect("Unknown user timer token"); - if (*timer.handler)(&mut HostIo::new(timer.protocol, None, event_loop, &mut self.channel)) { - event_loop.timeout_ms(token, timer.delay as u64).expect("Unable to reregister user timer"); + let protocol = self.timers.get_mut(token).expect("Unknown user timer token").protocol; + match self.handlers.get_mut(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(h) => h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()), } } _ => panic!("Unknown timer token"), @@ -632,22 +652,6 @@ impl Handler for Host { } } }, - HostMessage::AddTimer { - handler, - delay, - protocol, - } => { - match self.timers.insert(UserTimer { - handler: handler, - delay: delay, - protocol: protocol, - }) { - Ok(token) => { - event_loop.timeout_ms(token, delay as u64).expect("Error registering user timer"); - }, - _ => { panic!("Max timers reached") } - } - } } } } diff --git a/src/network/mod.rs b/src/network/mod.rs index 0b908a417..62928b020 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -47,6 +47,7 @@ impl From<::crypto::CryptoError> for Error { Error::Crypto(err) } } + impl From<::std::net::AddrParseError> for Error { fn from(err: ::std::net::AddrParseError) -> Error { Error::AddressParse(err) @@ -70,15 +71,19 @@ impl From<::mio::NotifyError> for Error { } pub type PeerId = host::PeerId; +pub type PacketId = host::PacketId; +pub type TimerToken = host::TimerToken; pub type HandlerIo<'s> = host::HostIo<'s>; pub trait ProtocolHandler: Send { fn initialize(&mut self, io: &mut HandlerIo); - fn read(&mut self, io: &mut HandlerIo, packet_id: u8, data: &[u8]); + fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]); fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId); fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); + fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken); } pub struct NetworkClient; +pub type NetworkService = service::NetworkService; From 88debb21901a8ce401e4cdf8ea6ec73c3ccfcc15 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 24 Dec 2015 00:31:43 +0100 Subject: [PATCH 277/379] address to and from h256 --- src/hash.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/hash.rs b/src/hash.rs index e86f8e834..ea041f2ac 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -320,6 +320,26 @@ impl<'a> From<&'a U256> for H256 { } } +impl From for Address { + fn from(value: H256) -> Address { + unsafe { + let mut ret: Address = ::std::mem::uninitialized(); + ::std::ptr::copy(value.as_ptr().offset(12), ret.as_mut_ptr(), 20); + ret + } + } +} + +impl From
for H256 { + fn from(value: Address) -> H256 { + unsafe { + let mut ret = H256::new(); + ::std::ptr::copy(value.as_ptr(), ret.as_mut_ptr().offset(12), 20); + ret + } + } +} + impl_hash!(H32, 4); impl_hash!(H64, 8); impl_hash!(H128, 16); @@ -382,5 +402,13 @@ mod tests { assert!(my_bloom.contains_bloom(&address.sha3())); assert!(my_bloom.contains_bloom(&topic.sha3())); } + + #[test] + fn from_and_to_address() { + let address = Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(); + let h = H256::from(address.clone()); + let a = Address::from(h); + assert_eq!(address, a); + } } From fa1b74fa5355352e5c90dc770e9431111a3a2774 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 26 Dec 2015 15:48:41 +0100 Subject: [PATCH 278/379] minor fixes --- src/bytes.rs | 39 ++++++++++++++++++------- src/hash.rs | 4 +-- src/rlp/mod.rs | 18 +++++++----- src/rlp/rlptraits.rs | 63 ++++++++++++++++++++-------------------- src/rlp/untrusted_rlp.rs | 4 +++ src/sha3.rs | 8 +++-- src/uint.rs | 24 +++++++++++++-- 7 files changed, 103 insertions(+), 57 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index d188eb69b..2cbfb00b4 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -1,27 +1,27 @@ //! Unified interfaces for bytes operations on basic types -//! +//! //! # Examples //! ```rust //! extern crate ethcore_util as util; -//! +//! //! fn bytes_convertable() { //! use util::bytes::BytesConvertable; //! //! let arr = [0; 5]; //! let slice: &[u8] = arr.bytes(); //! } -//! +//! //! fn to_bytes() { //! use util::bytes::ToBytes; -//! +//! //! let a: Vec = "hello_world".to_bytes(); //! let b: Vec = 400u32.to_bytes(); //! let c: Vec = 0xffffffffffffffffu64.to_bytes(); //! } -//! +//! //! fn from_bytes() { //! use util::bytes::FromBytes; -//! +//! //! let a = String::from_bytes(&[b'd', b'o', b'g']); //! let b = u16::from_bytes(&[0xfa]); //! let c = u64::from_bytes(&[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]); @@ -138,7 +138,7 @@ impl <'a> ToBytes for &'a str { fn to_bytes(&self) -> Vec { From::from(*self) } - + fn to_bytes_len(&self) -> usize { self.len() } } @@ -147,7 +147,7 @@ impl ToBytes for String { let s: &str = self.as_ref(); From::from(s) } - + fn to_bytes_len(&self) -> usize { self.len() } } @@ -166,6 +166,14 @@ impl ToBytes for u64 { fn to_bytes_len(&self) -> usize { 8 - self.leading_zeros() as usize / 8 } } +impl ToBytes for bool { + fn to_bytes(&self) -> Vec { + vec![ if *self { 1u8 } else { 0u8 } ] + } + + fn to_bytes_len(&self) -> usize { 1 } +} + macro_rules! impl_map_to_bytes { ($from: ident, $to: ty) => { impl ToBytes for $from { @@ -182,7 +190,7 @@ impl_map_to_bytes!(u32, u64); macro_rules! impl_uint_to_bytes { ($name: ident) => { impl ToBytes for $name { - fn to_bytes(&self) -> Vec { + fn to_bytes(&self) -> Vec { let mut res= vec![]; let count = self.to_bytes_len(); res.reserve(count); @@ -210,7 +218,7 @@ impl ToBytes for T where T: FixedHash { ptr::copy(self.bytes().as_ptr(), res.as_mut_ptr(), T::size()); res.set_len(T::size()); } - + res } } @@ -264,6 +272,17 @@ impl FromBytes for u64 { } } +impl FromBytes for bool { + fn from_bytes(bytes: &[u8]) -> FromBytesResult { + match bytes.len() { + 0 => Ok(false), + 1 => Ok(bytes[0] != 0), + _ => Err(FromBytesError::DataIsTooLong), + } + } +} + + macro_rules! impl_map_from_bytes { ($from: ident, $to: ident) => { impl FromBytes for $from { diff --git a/src/hash.rs b/src/hash.rs index 5ead8ab11..6d2c4de95 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -14,7 +14,7 @@ use math::log2; use uint::U256; /// Trait for a fixed-size byte array to be used as the output of hash functions. -/// +/// /// Note: types implementing `FixedHash` must be also `BytesConvertable`. pub trait FixedHash: Sized + BytesConvertable { fn new() -> Self; @@ -33,7 +33,7 @@ pub trait FixedHash: Sized + BytesConvertable { macro_rules! impl_hash { ($from: ident, $size: expr) => { - #[derive(Eq)] + #[derive(Eq, Copy)] pub struct $from (pub [u8; $size]); impl BytesConvertable for $from { diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 9c15259bd..8aa816c04 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -1,10 +1,10 @@ -//! Rlp serialization module -//! -//! Allows encoding, decoding, and view onto rlp-slice +//! Rlp serialization module +//! +//! Allows encoding, decoding, and view onto rlp-slice //! //!# What should you use when? //! -//!### Use `encode` function when: +//!### Use `encode` function when: //! * You want to encode something inline. //! * You do not work on big set of data. //! * You want to encode whole data structure at once. @@ -23,7 +23,7 @@ //! * You want to get view onto rlp-slice. //! * You don't want to decode whole rlp at once. //! -//!### Use `UntrustedRlp` when: +//!### Use `UntrustedRlp` when: //! * You are working on untrusted data (~corrupted). //! * You need to handle data corruption errors. //! * You are working on input data. @@ -47,14 +47,16 @@ pub use self::rlpstream::{RlpStream}; use super::hash::H256; pub const NULL_RLP: [u8; 1] = [0x80; 1]; +pub const EMPTY_LIST_RLP: [u8; 1] = [0xC0; 1]; pub const SHA3_NULL_RLP: H256 = H256( [0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21] ); +pub const SHA3_EMPTY_LIST_RLP: H256 = H256( [0x1d, 0xcc, 0x4d, 0xe8, 0xde, 0xc7, 0x5d, 0x7a, 0xab, 0x85, 0xb5, 0x67, 0xb6, 0xcc, 0xd4, 0x1a, 0xd3, 0x12, 0x45, 0x1b, 0x94, 0x8a, 0x74, 0x13, 0xf0, 0xa1, 0x42, 0xfd, 0x40, 0xd4, 0x93, 0x47] ); /// Shortcut function to decode trusted rlp -/// +/// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; -/// +/// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let animals: Vec = decode(&data); @@ -71,7 +73,7 @@ pub fn decode(bytes: &[u8]) -> T where T: Decodable { /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; -/// +/// /// fn main () { /// let animals = vec!["cat", "dog"]; /// let out = encode(&animals); diff --git a/src/rlp/rlptraits.rs b/src/rlp/rlptraits.rs index 067c438bf..26b663811 100644 --- a/src/rlp/rlptraits.rs +++ b/src/rlp/rlptraits.rs @@ -1,10 +1,11 @@ -use rlp::DecoderError; +use rlp::{DecoderError, UntrustedRlp}; pub trait Decoder: Sized { fn read_value(&self, f: F) -> Result where F: FnOnce(&[u8]) -> Result; fn as_list(&self) -> Result, DecoderError>; + fn as_rlp<'a>(&'a self) -> &'a UntrustedRlp<'a>; } pub trait Decodable: Sized { @@ -22,11 +23,11 @@ pub trait View<'a, 'view>: Sized { fn new(bytes: &'a [u8]) -> Self; /// The raw data of the RLP. - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -34,7 +35,7 @@ pub trait View<'a, 'view>: Sized { /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - fn raw(&'view self) -> &'a [u8]; + fn raw(&'view self) -> &'a [u8]; /// Get the prototype of the RLP. fn prototype(&self) -> Self::Prototype; @@ -44,11 +45,11 @@ pub trait View<'a, 'view>: Sized { fn data(&'view self) -> Self::Data; /// Returns number of RLP items. - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -60,11 +61,11 @@ pub trait View<'a, 'view>: Sized { fn item_count(&self) -> usize; /// Returns the number of bytes in the data, or zero if it isn't data. - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -76,14 +77,14 @@ pub trait View<'a, 'view>: Sized { fn size(&self) -> usize; /// Get view onto RLP-slice at index. - /// + /// /// Caches offset to given index, so access to successive /// slices is faster. - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -93,11 +94,11 @@ pub trait View<'a, 'view>: Sized { fn at(&'view self, index: usize) -> Self::Item; /// No value - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![]; /// let rlp = Rlp::new(&data); @@ -107,11 +108,11 @@ pub trait View<'a, 'view>: Sized { fn is_null(&self) -> bool; /// Contains a zero-length string or zero-length list. - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc0]; /// let rlp = Rlp::new(&data); @@ -121,11 +122,11 @@ pub trait View<'a, 'view>: Sized { fn is_empty(&self) -> bool; /// List value - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -135,11 +136,11 @@ pub trait View<'a, 'view>: Sized { fn is_list(&self) -> bool; /// String value - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -149,11 +150,11 @@ pub trait View<'a, 'view>: Sized { fn is_data(&self) -> bool; /// Int value - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc1, 0x10]; /// let rlp = Rlp::new(&data); @@ -164,11 +165,11 @@ pub trait View<'a, 'view>: Sized { fn is_int(&self) -> bool; /// Get iterator over rlp-slices - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); @@ -204,7 +205,7 @@ pub trait Stream: Sized { /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let mut stream = RlpStream::new_list(2); /// stream.append(&"cat").append(&"dog"); @@ -219,11 +220,11 @@ pub trait Stream: Sized { /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let mut stream = RlpStream::new_list(2); /// stream.append_list(2).append(&"cat").append(&"dog"); - /// stream.append(&""); + /// stream.append(&""); /// let out = stream.out(); /// assert_eq!(out, vec![0xca, 0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g', 0x80]); /// } @@ -235,7 +236,7 @@ pub trait Stream: Sized { /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let mut stream = RlpStream::new_list(2); /// stream.append_empty_data().append_empty_data(); @@ -249,11 +250,11 @@ pub trait Stream: Sized { fn append_raw<'a>(&'a mut self, bytes: &[u8], item_count: usize) -> &'a mut Self; /// Clear the output stream so far. - /// + /// /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let mut stream = RlpStream::new_list(3); /// stream.append(&"cat"); @@ -269,7 +270,7 @@ pub trait Stream: Sized { /// ```rust /// extern crate ethcore_util as util; /// use util::rlp::*; - /// + /// /// fn main () { /// let mut stream = RlpStream::new_list(2); /// stream.append(&"cat"); @@ -284,7 +285,7 @@ pub trait Stream: Sized { fn raw(&self) -> &[u8]; /// Streams out encoded bytes. - /// + /// /// panic! if stream is not finished. fn out(self) -> Vec; } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index d88563c30..b2c7784e7 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -321,6 +321,10 @@ impl<'a> Decoder for BasicDecoder<'a> { .collect(); Ok(v) } + + fn as_rlp<'s>(&'s self) -> &'s UntrustedRlp<'s> { + &self.rlp + } } impl Decodable for T where T: FromBytes { diff --git a/src/sha3.rs b/src/sha3.rs index f7a298c4a..cea953f5c 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -6,20 +6,22 @@ use bytes::BytesConvertable; use hash::{FixedHash, H256}; /// Types implementing this trait are sha3able. -/// +/// /// ``` /// extern crate ethcore_util as util; /// use std::str::FromStr; /// use util::sha3::*; /// use util::hash::*; -/// +/// /// fn main() { /// assert_eq!([0u8; 0].sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); /// } /// ``` pub trait Hashable { fn sha3(&self) -> H256; - fn sha3_into(&self, dest: &mut [u8]); + fn sha3_into(&self, dest: &mut [u8]) { + self.sha3().copy_to(dest); + } } impl Hashable for T where T: BytesConvertable { diff --git a/src/uint.rs b/src/uint.rs index b8eccc4cc..6b2af4b3b 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -52,11 +52,17 @@ macro_rules! construct_uint { impl $name { /// Conversion to u32 #[inline] - fn low_u32(&self) -> u32 { + pub fn low_u32(&self) -> u32 { let &$name(ref arr) = self; arr[0] as u32 } + /// Conversion to u64 + #[inline] + pub fn low_u64(&self) -> u64 { + let &$name(ref arr) = self; + arr[0] + } /// Return the least number of bits needed to represent the number #[inline] pub fn bits(&self) -> usize { @@ -101,7 +107,7 @@ macro_rules! construct_uint { pub fn zero() -> $name { From::from(0u64) } - + #[inline] pub fn one() -> $name { From::from(1u64) @@ -410,7 +416,7 @@ macro_rules! construct_uint { fn from_dec_str(value: &str) -> Result { Ok(value.bytes() .map(|b| b - 48) - .fold($name::from(0u64), | acc, c | + .fold($name::from(0u64), | acc, c | // fast multiplication by 10 // (acc << 3) + (acc << 1) => acc * 10 (acc << 3) + (acc << 1) + $name::from(c) @@ -434,6 +440,18 @@ impl From for U256 { } } +impl From for u64 { + fn from(value: U256) -> u64 { + value.low_u64() + } +} + +impl From for u32 { + fn from(value: U256) -> u32 { + value.low_u32() + } +} + pub const ZERO_U256: U256 = U256([0x00u64; 4]); pub const ONE_U256: U256 = U256([0x01u64, 0x00u64, 0x00u64, 0x00u64]); pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); From 634b6be1e666e7b93295641ba501877d081983d5 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 26 Dec 2015 15:50:55 +0100 Subject: [PATCH 279/379] fixed warnings --- src/network/host.rs | 16 ---------------- src/rlp/tests.rs | 6 +++--- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index 68430c688..52b635f55 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -655,19 +655,3 @@ impl Handler for Host { } } } - - -#[cfg(test)] -mod tests { - use network::host::Host; - use env_logger; - #[test] - //#[ignore] - fn net_connect() { - env_logger::init().unwrap(); - let _ = Host::start(); - } -} - - - diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs index b4a60a3de..49698dafb 100644 --- a/src/rlp/tests.rs +++ b/src/rlp/tests.rs @@ -4,7 +4,7 @@ use self::json_tests::rlp as rlptest; use std::{fmt, cmp}; use std::str::FromStr; use rlp; -use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; +use rlp::{UntrustedRlp, RlpStream, View, Stream}; use uint::U256; #[test] @@ -155,7 +155,7 @@ fn encode_address() { use hash::*; let tests = vec![ - ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + ETestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, 0xb3, 0x7d, 0x11, 0x06]) @@ -290,7 +290,7 @@ fn decode_untrusted_address() { use hash::*; let tests = vec![ - DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), + DTestPair(Address::from_str("ef2d6d194084c2de36e0dabfce45d046b37d1106").unwrap(), vec![0x94, 0xef, 0x2d, 0x6d, 0x19, 0x40, 0x84, 0xc2, 0xde, 0x36, 0xe0, 0xda, 0xbf, 0xce, 0x45, 0xd0, 0x46, 0xb3, 0x7d, 0x11, 0x06]) From 267495c26481bc7c19c890396a78a14a236357f8 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 28 Dec 2015 11:41:51 +0100 Subject: [PATCH 280/379] User IO messages --- src/network/host.rs | 28 ++++++++++++++++++++++++++++ src/network/mod.rs | 3 +++ 2 files changed, 31 insertions(+) diff --git a/src/network/host.rs b/src/network/host.rs index 52b635f55..50954661c 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -162,6 +162,15 @@ pub enum HostMessage { protocol: ProtocolId, data: Vec, }, + UserMessage(UserMessage), +} + +pub type UserMessageId = u32; + +pub struct UserMessage { + protocol: ProtocolId, + id: UserMessageId, + data: Option>, } pub type PeerId = usize; @@ -237,9 +246,21 @@ impl<'s> HostIo<'s> { } } + pub fn message(&mut self, id: UserMessageId, data: Option>) { + match self.event_loop.channel().send(HostMessage::UserMessage(UserMessage { + protocol: self.protocol, + id: id, + data: data + })) { + Ok(_) => {} + Err(e) => { panic!("Error sending io message {:?}", e); } + } + } + pub fn disable_peer(&mut self, _peer: PeerId) { //TODO: remove capability, disconnect if no capabilities left } + } struct UserTimer { @@ -652,6 +673,13 @@ impl Handler for Host { } } }, + HostMessage::UserMessage(message) => { + for (p, h) in self.handlers.iter_mut() { + if p != &message.protocol { + h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); + } + } + } } } } diff --git a/src/network/mod.rs b/src/network/mod.rs index 62928b020..b3e922172 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -74,6 +74,8 @@ pub type PeerId = host::PeerId; pub type PacketId = host::PacketId; pub type TimerToken = host::TimerToken; pub type HandlerIo<'s> = host::HostIo<'s>; +pub type Message = host::UserMessage; +pub type MessageId = host::UserMessageId; pub trait ProtocolHandler: Send { fn initialize(&mut self, io: &mut HandlerIo); @@ -81,6 +83,7 @@ pub trait ProtocolHandler: Send { fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId); fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken); + fn message(&mut self, io: &mut HandlerIo, message: &Message); } pub struct NetworkClient; From 9e0c8e3d8bb3be793b459707dd7f7f599b8be7e7 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 30 Dec 2015 12:23:36 +0100 Subject: [PATCH 281/379] Documentation --- src/network/host.rs | 6 ++++++ src/network/mod.rs | 11 +++++++++-- src/network/service.rs | 4 ++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index 50954661c..3d8e78e46 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -191,6 +191,7 @@ impl Encodable for CapabilityInfo { } } +/// IO access point pub struct HostIo<'s> { protocol: ProtocolId, connections: &'s mut Slab, @@ -210,6 +211,7 @@ impl<'s> HostIo<'s> { } } + /// Send a packet over the network to another peer. pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), Error> { match self.connections.get_mut(Token(peer)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { @@ -224,6 +226,7 @@ impl<'s> HostIo<'s> { Ok(()) } + /// Respond to a current network message. Panics if no there is no packet in the context. pub fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), Error> { match self.session { Some(session) => self.send(session.as_usize(), packet_id, data), @@ -233,6 +236,7 @@ impl<'s> HostIo<'s> { } } + /// Register a new IO timer. Returns a new timer toke. 'ProtocolHandler::timeout' will be called with the token. pub fn register_timer(&mut self, ms: u64) -> Result{ match self.timers.insert(UserTimer { delay: ms, @@ -246,6 +250,7 @@ impl<'s> HostIo<'s> { } } + /// Broadcast a message to other IO clients pub fn message(&mut self, id: UserMessageId, data: Option>) { match self.event_loop.channel().send(HostMessage::UserMessage(UserMessage { protocol: self.protocol, @@ -257,6 +262,7 @@ impl<'s> HostIo<'s> { } } + /// Disable current protocol capability for given peer. If no capabilities left peer gets disconnected. pub fn disable_peer(&mut self, _peer: PeerId) { //TODO: remove capability, disconnect if no capabilities left } diff --git a/src/network/mod.rs b/src/network/mod.rs index b3e922172..c44182430 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -77,16 +77,23 @@ pub type HandlerIo<'s> = host::HostIo<'s>; pub type Message = host::UserMessage; pub type MessageId = host::UserMessageId; +/// Network IO protocol handler. This needs to be implemented for each new subprotocol. +/// TODO: Separate p2p networking IO from IPC IO. `timeout` and `message` should go to a more genera IO provider. +/// All the handler function are called from within IO event loop. pub trait ProtocolHandler: Send { + /// Initialize the hadler fn initialize(&mut self, io: &mut HandlerIo); + /// Called when new network packet received. fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]); + /// Called when new peer is connected. Only called when peer supports the same protocol. fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId); + /// Called when a previously connected peer disconnects. fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); + /// Timer function called after a timeout created with `HandlerIo::timeout`. fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken); + /// Called when a broadcasted message is received. The message can only be sent from a different protocol handler. fn message(&mut self, io: &mut HandlerIo, message: &Message); } -pub struct NetworkClient; pub type NetworkService = service::NetworkService; - diff --git a/src/network/service.rs b/src/network/service.rs index 1e245c20b..986d6dd11 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -5,12 +5,14 @@ use mio::*; use network::{Error, ProtocolHandler}; use network::host::{Host, HostMessage, PeerId, PacketId, ProtocolId}; +/// IO Service with networking pub struct NetworkService { thread: Option>, host_channel: Sender } impl NetworkService { + /// Starts IO event loop pub fn start() -> Result { let mut event_loop = EventLoop::new().unwrap(); let channel = event_loop.channel(); @@ -23,6 +25,7 @@ impl NetworkService { }) } + /// Send a message over the network. Normaly `HostIo::send` should be used. This can be used from non-io threads. pub fn send(&mut self, peer: &PeerId, packet_id: PacketId, protocol: ProtocolId, data: &[u8]) -> Result<(), Error> { try!(self.host_channel.send(HostMessage::Send { peer: *peer, @@ -33,6 +36,7 @@ impl NetworkService { Ok(()) } + /// Regiter a new protocol handler with the event loop. pub fn register_protocol(&mut self, handler: Box, protocol: ProtocolId, versions: &[u8]) -> Result<(), Error> { try!(self.host_channel.send(HostMessage::AddHandler { handler: handler, From 8be769d9281ab161f2bac9cf39d11f9cd911a2f7 Mon Sep 17 00:00:00 2001 From: debris Date: Thu, 31 Dec 2015 12:24:30 +0100 Subject: [PATCH 282/379] fix rocksdb version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4d23f49f8..486cc6a8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ mio = "0.4.4" rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" -rocksdb = "0.2.1" +rocksdb = "0.2" lazy_static = "0.1.*" secp256k1 = "0.5.1" rust-crypto = "0.2.34" From 32fcc6b3e91c5b70c8949153b6707a49aeacdcad Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 3 Jan 2016 09:23:45 +0100 Subject: [PATCH 283/379] Removed fmt file --- src/network/.host.rs.rustfmt | 685 ----------------------------------- 1 file changed, 685 deletions(-) delete mode 100644 src/network/.host.rs.rustfmt diff --git a/src/network/.host.rs.rustfmt b/src/network/.host.rs.rustfmt deleted file mode 100644 index 50954661c..000000000 --- a/src/network/.host.rs.rustfmt +++ /dev/null @@ -1,685 +0,0 @@ -#![allow(dead_code)] //TODO: remove this after everything is done -//TODO: remove all unwraps -use std::net::{SocketAddr, ToSocketAddrs}; -use std::collections::{HashMap}; -use std::hash::{Hash, Hasher}; -use std::str::{FromStr}; -use mio::*; -use mio::util::{Slab}; -use mio::tcp::*; -use mio::udp::*; -use hash::*; -use crypto::*; -use rlp::*; -use time::Tm; -use network::handshake::Handshake; -use network::session::{Session, SessionData}; -use network::{Error, ProtocolHandler}; - -const DEFAULT_PORT: u16 = 30304; - -const MAX_CONNECTIONS: usize = 1024; -const MAX_USER_TIMERS: usize = 32; -const IDEAL_PEERS:u32 = 10; - -pub type NodeId = H512; -pub type TimerToken = usize; - -#[derive(Debug)] -struct NetworkConfiguration { - listen_address: SocketAddr, - public_address: SocketAddr, - no_nat: bool, - no_discovery: bool, - pin: bool, -} - -impl NetworkConfiguration { - fn new() -> NetworkConfiguration { - NetworkConfiguration { - listen_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), - public_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), - no_nat: false, - no_discovery: false, - pin: false, - } - } -} - -#[derive(Debug)] -pub struct NodeEndpoint { - address: SocketAddr, - address_str: String, - udp_port: u16 -} - -impl NodeEndpoint { - fn new(address: SocketAddr) -> NodeEndpoint { - NodeEndpoint { - address: address, - address_str: address.to_string(), - udp_port: address.port() - } - } - fn from_str(s: &str) -> Result { - println!("{:?}", s); - let address = s.to_socket_addrs().map(|mut i| i.next()); - match address { - Ok(Some(a)) => Ok(NodeEndpoint { - address: a, - address_str: s.to_string(), - udp_port: a.port() - }), - Ok(_) => Err(Error::AddressResolve(None)), - Err(e) => Err(Error::AddressResolve(Some(e))) - } - } -} - -#[derive(PartialEq, Eq, Copy, Clone)] -enum PeerType { - Required, - Optional -} - -struct Node { - id: NodeId, - endpoint: NodeEndpoint, - peer_type: PeerType, - last_attempted: Option, - confirmed: bool, -} - -impl FromStr for Node { - type Err = Error; - fn from_str(s: &str) -> Result { - let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { - (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) - } - else { - (NodeId::new(), try!(NodeEndpoint::from_str(s))) - }; - - Ok(Node { - id: id, - endpoint: endpoint, - peer_type: PeerType::Optional, - last_attempted: None, - confirmed: false - }) - } -} - -impl Node { - fn new(id: NodeId, address: SocketAddr, t:PeerType) -> Node { - Node { - id: id, - endpoint: NodeEndpoint::new(address), - peer_type: t, - last_attempted: None, - confirmed: false - } - } -} - -impl PartialEq for Node { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } -} -impl Eq for Node { } - -impl Hash for Node { - fn hash(&self, state: &mut H) where H: Hasher { - self.id.hash(state) - } -} - -// Tokens -const TCP_ACCEPT: usize = 1; -const IDLE: usize = 3; -const NODETABLE_RECEIVE: usize = 4; -const NODETABLE_MAINTAIN: usize = 5; -const NODETABLE_DISCOVERY: usize = 6; -const FIRST_CONNECTION: usize = 7; -const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; -const USER_TIMER: usize = LAST_CONNECTION; -const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; - -pub type PacketId = u8; -pub type ProtocolId = &'static str; - -pub enum HostMessage { - Shutdown, - AddHandler { - handler: Box, - protocol: ProtocolId, - versions: Vec, - }, - Send { - peer: PeerId, - packet_id: PacketId, - protocol: ProtocolId, - data: Vec, - }, - UserMessage(UserMessage), -} - -pub type UserMessageId = u32; - -pub struct UserMessage { - protocol: ProtocolId, - id: UserMessageId, - data: Option>, -} - -pub type PeerId = usize; - -#[derive(Debug, PartialEq, Eq)] -pub struct CapabilityInfo { - pub protocol: ProtocolId, - pub version: u8, - pub packet_count: u8, -} - -impl Encodable for CapabilityInfo { - fn encode(&self, encoder: &mut E) -> () where E: Encoder { - encoder.emit_list(|e| { - self.protocol.encode(e); - (self.version as u32).encode(e); - }); - } -} - -pub struct HostIo<'s> { - protocol: ProtocolId, - connections: &'s mut Slab, - timers: &'s mut Slab, - session: Option, - event_loop: &'s mut EventLoop, -} - -impl<'s> HostIo<'s> { - fn new(protocol: ProtocolId, session: Option, event_loop: &'s mut EventLoop, connections: &'s mut Slab, timers: &'s mut Slab) -> HostIo<'s> { - HostIo { - protocol: protocol, - session: session, - event_loop: event_loop, - connections: connections, - timers: timers, - } - } - - pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), Error> { - match self.connections.get_mut(Token(peer)) { - Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { - warn!(target: "net", "Send error: {:?}", e); - }); //TODO: don't copy vector data - }, - _ => { - warn!(target: "net", "Send: Peer does not exist"); - } - } - Ok(()) - } - - pub fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), Error> { - match self.session { - Some(session) => self.send(session.as_usize(), packet_id, data), - None => { - panic!("Respond: Session does not exist") - } - } - } - - pub fn register_timer(&mut self, ms: u64) -> Result{ - match self.timers.insert(UserTimer { - delay: ms, - protocol: self.protocol, - }) { - Ok(token) => { - self.event_loop.timeout_ms(token, ms).expect("Error registering user timer"); - Ok(token.as_usize()) - }, - _ => { panic!("Max timers reached") } - } - } - - pub fn message(&mut self, id: UserMessageId, data: Option>) { - match self.event_loop.channel().send(HostMessage::UserMessage(UserMessage { - protocol: self.protocol, - id: id, - data: data - })) { - Ok(_) => {} - Err(e) => { panic!("Error sending io message {:?}", e); } - } - } - - pub fn disable_peer(&mut self, _peer: PeerId) { - //TODO: remove capability, disconnect if no capabilities left - } - -} - -struct UserTimer { - protocol: ProtocolId, - delay: u64, -} - -pub struct HostInfo { - keys: KeyPair, - config: NetworkConfiguration, - nonce: H256, - pub protocol_version: u32, - pub client_version: String, - pub listen_port: u16, - pub capabilities: Vec -} - -impl HostInfo { - pub fn id(&self) -> &NodeId { - self.keys.public() - } - - pub fn secret(&self) -> &Secret { - self.keys.secret() - } - pub fn next_nonce(&mut self) -> H256 { - self.nonce = self.nonce.sha3(); - return self.nonce.clone(); - } -} - -enum ConnectionEntry { - Handshake(Handshake), - Session(Session) -} - -pub struct Host { - info: HostInfo, - udp_socket: UdpSocket, - listener: TcpListener, - connections: Slab, - timers: Slab, - nodes: HashMap, - handlers: HashMap>, - idle_timeout: Timeout, - channel: Sender, -} - -impl Host { - pub fn start(event_loop: &mut EventLoop) -> Result<(), Error> { - let config = NetworkConfiguration::new(); - /* - match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { - Some(iface) => config.public_address = iface.addr.unwrap(), - None => warn!("No public network interface"), - } - */ - - let addr = config.listen_address; - // Setup the server socket - let listener = TcpListener::bind(&addr).unwrap(); - // Start listening for incoming connections - event_loop.register(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); - // Setup the client socket - //let sock = TcpStream::connect(&addr).unwrap(); - // Register the socket - //self.event_loop.register(&sock, CLIENT, EventSet::readable(), PollOpt::edge()).unwrap(); - let idle_timeout = event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay - // open the udp socket - let udp_socket = UdpSocket::bound(&addr).unwrap(); - event_loop.register(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); - event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); - let port = config.listen_address.port(); - - let mut host = Host { - info: HostInfo { - keys: KeyPair::create().unwrap(), - config: config, - nonce: H256::random(), - protocol_version: 4, - client_version: "parity".to_string(), - listen_port: port, - //capabilities: vec![ CapabilityInfo { protocol: "eth".to_string(), version: 63 }], - capabilities: Vec::new(), - }, - udp_socket: udp_socket, - listener: listener, - connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), - timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), - nodes: HashMap::new(), - handlers: HashMap::new(), - idle_timeout: idle_timeout, - channel: event_loop.channel(), - }; - - host.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); - host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); - host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); - host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); - host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); - host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); - - try!(event_loop.run(&mut host)); - Ok(()) - } - - fn add_node(&mut self, id: &str) { - match Node::from_str(id) { - Err(e) => { warn!("Could not add node: {:?}", e); }, - Ok(n) => { - self.nodes.insert(n.id.clone(), n); - } - } - } - - fn maintain_network(&mut self, event_loop: &mut EventLoop) { - self.connect_peers(event_loop); - } - - fn have_session(&self, id: &NodeId) -> bool { - self.connections.iter().any(|e| match e { &ConnectionEntry::Session(ref s) => s.info.id.eq(&id), _ => false }) - } - - fn connecting_to(&self, id: &NodeId) -> bool { - self.connections.iter().any(|e| match e { &ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) - } - - fn connect_peers(&mut self, event_loop: &mut EventLoop) { - - struct NodeInfo { - id: NodeId, - peer_type: PeerType - } - - let mut to_connect: Vec = Vec::new(); - - let mut req_conn = 0; - //TODO: use nodes from discovery here - //for n in self.node_buckets.iter().flat_map(|n| &n.nodes).map(|id| NodeInfo { id: id.clone(), peer_type: self.nodes.get(id).unwrap().peer_type}) { - for n in self.nodes.values().map(|n| NodeInfo { id: n.id.clone(), peer_type: n.peer_type }) { - let connected = self.have_session(&n.id) || self.connecting_to(&n.id); - let required = n.peer_type == PeerType::Required; - if connected && required { - req_conn += 1; - } - else if !connected && (!self.info.config.pin || required) { - to_connect.push(n); - } - } - - for n in to_connect.iter() { - if n.peer_type == PeerType::Required { - if req_conn < IDEAL_PEERS { - self.connect_peer(&n.id, event_loop); - } - req_conn += 1; - } - } - - if !self.info.config.pin - { - let pending_count = 0; //TODO: - let peer_count = 0; - let mut open_slots = IDEAL_PEERS - peer_count - pending_count + req_conn; - if open_slots > 0 { - for n in to_connect.iter() { - if n.peer_type == PeerType::Optional && open_slots > 0 { - open_slots -= 1; - self.connect_peer(&n.id, event_loop); - } - } - } - } - } - - fn connect_peer(&mut self, id: &NodeId, event_loop: &mut EventLoop) { - if self.have_session(id) - { - warn!("Aborted connect. Node already connected."); - return; - } - if self.connecting_to(id) - { - warn!("Aborted connect. Node already connecting."); - return; - } - - let socket = { - let node = self.nodes.get_mut(id).unwrap(); - node.last_attempted = Some(::time::now()); - - - //blog(NetConnect) << "Attempting connection to node" << _p->id << "@" << ep << "from" << id(); - match TcpStream::connect(&node.endpoint.address) { - Ok(socket) => socket, - Err(_) => { - warn!("Cannot connect to node"); - return; - } - } - }; - - let nonce = self.info.next_nonce(); - match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(token, id, socket, &nonce).expect("Can't create handshake"))) { - Some(token) => { - match self.connections.get_mut(token) { - Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.start(&self.info, true) - .and_then(|_| h.register(event_loop)) - .unwrap_or_else (|e| { - debug!(target: "net", "Handshake create error: {:?}", e); - }); - }, - _ => {} - } - }, - None => { warn!("Max connections reached") } - } - } - - - fn accept(&mut self, _event_loop: &mut EventLoop) { - warn!(target: "net", "accept"); - } - - fn connection_writable(&mut self, token: Token, event_loop: &mut EventLoop) { - let mut kill = false; - let mut create_session = false; - { - match self.connections.get_mut(token) { - Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.writable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Handshake write error: {:?}", e); - kill = true; - }); - create_session = h.done(); - }, - Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.writable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Session write error: {:?}", e); - kill = true; - }); - } - _ => { - warn!(target: "net", "Received event for unknown connection"); - } - }; - } - if kill { - self.kill_connection(token, event_loop); - } - if create_session { - self.start_session(token, event_loop); - } - } - - - fn connection_readable(&mut self, token: Token, event_loop: &mut EventLoop) { - let mut kill = false; - let mut create_session = false; - let mut ready_data: Vec = Vec::new(); - let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; - { - match self.connections.get_mut(token) { - Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.readable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Handshake read error: {:?}", e); - kill = true; - }); - create_session = h.done(); - }, - Some(&mut ConnectionEntry::Session(ref mut s)) => { - let sd = { s.readable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Session read error: {:?}", e); - kill = true; - SessionData::None - }) }; - match sd { - SessionData::Ready => { - for (p, _) in self.handlers.iter_mut() { - if s.have_capability(p) { - ready_data.push(p); - } - } - }, - SessionData::Packet { - data, - protocol, - packet_id, - } => { - match self.handlers.get_mut(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(_) => packet_data = Some((protocol, packet_id, data)), - } - }, - SessionData::None => {}, - } - } - _ => { - warn!(target: "net", "Received event for unknown connection"); - } - }; - } - if kill { - self.kill_connection(token, event_loop); - } - if create_session { - self.start_session(token, event_loop); - } - for p in ready_data { - let mut h = self.handlers.get_mut(p).unwrap(); - h.connected(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize()); - } - if let Some((p, packet_id, data)) = packet_data { - let mut h = self.handlers.get_mut(p).unwrap(); - h.read(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize(), packet_id, &data[1..]); - } - - } - - fn start_session(&mut self, token: Token, event_loop: &mut EventLoop) { - let info = &self.info; - self.connections.replace_with(token, |c| { - match c { - ConnectionEntry::Handshake(h) => Session::new(h, event_loop, info) - .map(|s| Some(ConnectionEntry::Session(s))) - .unwrap_or_else(|e| { - debug!(target: "net", "Session construction error: {:?}", e); - None - }), - _ => { panic!("No handshake to create a session from"); } - } - }).expect("Error updating slab with session"); - } - - fn connection_timeout(&mut self, token: Token, event_loop: &mut EventLoop) { - self.kill_connection(token, event_loop) - } - fn kill_connection(&mut self, token: Token, _event_loop: &mut EventLoop) { - self.connections.remove(token); - } -} - -impl Handler for Host { - type Timeout = Token; - type Message = HostMessage; - - fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { - if events.is_readable() { - match token.as_usize() { - TCP_ACCEPT => self.accept(event_loop), - IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), - NODETABLE_RECEIVE => {}, - _ => panic!("Received unknown readable token"), - } - } - else if events.is_writable() { - match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), - _ => panic!("Received unknown writable token"), - } - } - } - - fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { - match token.as_usize() { - IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), - NODETABLE_DISCOVERY => {}, - NODETABLE_MAINTAIN => {}, - USER_TIMER ... LAST_USER_TIMER => { - let protocol = self.timers.get_mut(token).expect("Unknown user timer token").protocol; - match self.handlers.get_mut(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(h) => h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()), - } - } - _ => panic!("Unknown timer token"), - } - } - - fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { - match msg { - HostMessage::Shutdown => event_loop.shutdown(), - HostMessage::AddHandler { - handler, - protocol, - versions - } => { - self.handlers.insert(protocol, handler); - for v in versions { - self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); - } - }, - HostMessage::Send { - peer, - packet_id, - protocol, - data, - } => { - match self.connections.get_mut(Token(peer as usize)) { - Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { - warn!(target: "net", "Send error: {:?}", e); - }); //TODO: don't copy vector data - }, - _ => { - warn!(target: "net", "Send: Peer does not exist"); - } - } - }, - HostMessage::UserMessage(message) => { - for (p, h) in self.handlers.iter_mut() { - if p != &message.protocol { - h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); - } - } - } - } - } -} From 39f2dc9e2f80c269b05f551d85c98cc64db96059 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 4 Jan 2016 13:49:32 +0100 Subject: [PATCH 284/379] Style --- src/hash.rs | 2 +- src/network/discovery.rs | 295 ++++++++++++++++++++------------------- src/network/host.rs | 54 +++---- src/network/mod.rs | 2 +- src/network/service.rs | 2 - 5 files changed, 168 insertions(+), 187 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 6d2c4de95..995a6cf28 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -33,7 +33,7 @@ pub trait FixedHash: Sized + BytesConvertable { macro_rules! impl_hash { ($from: ident, $size: expr) => { - #[derive(Eq, Copy)] + #[derive(Eq)] pub struct $from (pub [u8; $size]); impl BytesConvertable for $from { diff --git a/src/network/discovery.rs b/src/network/discovery.rs index e8a342c22..bae52eb10 100644 --- a/src/network/discovery.rs +++ b/src/network/discovery.rs @@ -11,56 +11,57 @@ use hash::*; use crypto::*; use network::host::*; -const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. +const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia]. const NODE_BINS: u32 = ADDRESS_BITS - 1; ///< Size of m_state (excludes root, which is us). -const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover) -const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. +const DISCOVERY_MAX_STEPS: u16 = 8; ///< Max iterations of discovery. (discover) +const BUCKET_SIZE: u32 = 16; ///< Denoted by k in [Kademlia]. Number of nodes stored in each bucket. const ALPHA: usize = 3; ///< Denoted by \alpha in [Kademlia]. Number of concurrent FindNode requests. struct NodeBucket { - distance: u32, - nodes: Vec + distance: u32, + nodes: Vec } impl NodeBucket { - fn new(distance: u32) -> NodeBucket { - NodeBucket { - distance: distance, - nodes: Vec::new() - } - } + fn new(distance: u32) -> NodeBucket { + NodeBucket { + distance: distance, + nodes: Vec::new() + } + } } struct Discovery { - id: NodeId, - discovery_round: u16, - discovery_id: NodeId, - discovery_nodes: HashSet, - node_buckets: Vec, + id: NodeId, + discovery_round: u16, + discovery_id: NodeId, + discovery_nodes: HashSet, + node_buckets: Vec, } struct FindNodePacket; impl FindNodePacket { - fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { - FindNodePacket - } - fn sign(&mut self, _secret: &Secret) { - } + fn new(_endpoint: &NodeEndpoint, _id: &NodeId) -> FindNodePacket { + FindNodePacket + } - fn send(& self, _socket: &mut UdpSocket) { - } + fn sign(&mut self, _secret: &Secret) { + } + + fn send(& self, _socket: &mut UdpSocket) { + } } impl Discovery { pub fn new(id: &NodeId) -> Discovery { Discovery { id: id.clone(), - discovery_round: 0, - discovery_id: NodeId::new(), - discovery_nodes: HashSet::new(), - node_buckets: (0..NODE_BINS).map(|x| NodeBucket::new(x)).collect(), + discovery_round: 0, + discovery_id: NodeId::new(), + discovery_nodes: HashSet::new(), + node_buckets: (0..NODE_BINS).map(|x| NodeBucket::new(x)).collect(), } } @@ -68,137 +69,137 @@ impl Discovery { self.node_buckets[Discovery::distance(&self.id, &id) as usize].nodes.push(id.clone()); } - fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { - self.discovery_round = 0; - self.discovery_id.randomize(); - self.discovery_nodes.clear(); - self.discover(event_loop); - } + fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { + self.discovery_round = 0; + self.discovery_id.randomize(); + self.discovery_nodes.clear(); + self.discover(event_loop); + } - fn discover(&mut self, event_loop: &mut EventLoop) { - if self.discovery_round == DISCOVERY_MAX_STEPS - { - debug!("Restarting discovery"); - self.start_node_discovery(event_loop); - return; - } - let mut tried_count = 0; - { - let nearest = Discovery::nearest_node_entries(&self.id, &self.discovery_id, &self.node_buckets).into_iter(); - let nodes = RefCell::new(&mut self.discovery_nodes); - let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); - for r in nearest { - //let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id); - //p.sign(&self.secret); - //p.send(&mut self.udp_socket); - let mut borrowed = nodes.borrow_mut(); - borrowed.deref_mut().insert(r.clone()); - tried_count += 1; - } - } + fn discover(&mut self, event_loop: &mut EventLoop) { + if self.discovery_round == DISCOVERY_MAX_STEPS + { + debug!("Restarting discovery"); + self.start_node_discovery(event_loop); + return; + } + let mut tried_count = 0; + { + let nearest = Discovery::nearest_node_entries(&self.id, &self.discovery_id, &self.node_buckets).into_iter(); + let nodes = RefCell::new(&mut self.discovery_nodes); + let nearest = nearest.filter(|x| nodes.borrow().contains(&x)).take(ALPHA); + for r in nearest { + //let mut p = FindNodePacket::new(&r.endpoint, &self.discovery_id); + //p.sign(&self.secret); + //p.send(&mut self.udp_socket); + let mut borrowed = nodes.borrow_mut(); + borrowed.deref_mut().insert(r.clone()); + tried_count += 1; + } + } - if tried_count == 0 - { - debug!("Restarting discovery"); - self.start_node_discovery(event_loop); - return; - } - self.discovery_round += 1; - //event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap(); - } + if tried_count == 0 + { + debug!("Restarting discovery"); + self.start_node_discovery(event_loop); + return; + } + self.discovery_round += 1; + //event_loop.timeout_ms(Token(NODETABLE_DISCOVERY), 1200).unwrap(); + } fn distance(a: &NodeId, b: &NodeId) -> u32 { - let d = a.sha3() ^ b.sha3(); - let mut ret:u32 = 0; - for i in 0..32 { - let mut v: u8 = d[i]; - while v != 0 { - v >>= 1; - ret += 1; - } - } - ret - } + let d = a.sha3() ^ b.sha3(); + let mut ret:u32 = 0; + for i in 0..32 { + let mut v: u8 = d[i]; + while v != 0 { + v >>= 1; + ret += 1; + } + } + ret + } - fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b Vec) -> Vec<&'b NodeId> - { - // send ALPHA FindNode packets to nodes we know, closest to target - const LAST_BIN: u32 = NODE_BINS - 1; - let mut head = Discovery::distance(source, target); - let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; + fn nearest_node_entries<'b>(source: &NodeId, target: &NodeId, buckets: &'b Vec) -> Vec<&'b NodeId> + { + // send ALPHA FindNode packets to nodes we know, closest to target + const LAST_BIN: u32 = NODE_BINS - 1; + let mut head = Discovery::distance(source, target); + let mut tail = if head == 0 { LAST_BIN } else { (head - 1) % NODE_BINS }; - let mut found: BTreeMap> = BTreeMap::new(); - let mut count = 0; + let mut found: BTreeMap> = BTreeMap::new(); + let mut count = 0; - // if d is 0, then we roll look forward, if last, we reverse, else, spread from d - if head > 1 && tail != LAST_BIN { - while head != tail && head < NODE_BINS && count < BUCKET_SIZE - { - for n in buckets[head as usize].nodes.iter() - { - if count < BUCKET_SIZE { + // if d is 0, then we roll look forward, if last, we reverse, else, spread from d + if head > 1 && tail != LAST_BIN { + while head != tail && head < NODE_BINS && count < BUCKET_SIZE + { + for n in buckets[head as usize].nodes.iter() + { + if count < BUCKET_SIZE { count += 1; - found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - if count < BUCKET_SIZE && tail != 0 { - for n in buckets[tail as usize].nodes.iter() { - if count < BUCKET_SIZE { + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + if count < BUCKET_SIZE && tail != 0 { + for n in buckets[tail as usize].nodes.iter() { + if count < BUCKET_SIZE { count += 1; - found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - } + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + } - head += 1; - if tail > 0 { - tail -= 1; - } - } - } - else if head < 2 { - while head < NODE_BINS && count < BUCKET_SIZE { - for n in buckets[head as usize].nodes.iter() { - if count < BUCKET_SIZE { + head += 1; + if tail > 0 { + tail -= 1; + } + } + } + else if head < 2 { + while head < NODE_BINS && count < BUCKET_SIZE { + for n in buckets[head as usize].nodes.iter() { + if count < BUCKET_SIZE { count += 1; - found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - head += 1; - } - } - else { - while tail > 0 && count < BUCKET_SIZE { - for n in buckets[tail as usize].nodes.iter() { - if count < BUCKET_SIZE { + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + head += 1; + } + } + else { + while tail > 0 && count < BUCKET_SIZE { + for n in buckets[tail as usize].nodes.iter() { + if count < BUCKET_SIZE { count += 1; - found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); - } - else { - break; - } - } - tail -= 1; - } - } + found.entry(Discovery::distance(target, &n)).or_insert(Vec::new()).push(n); + } + else { + break; + } + } + tail -= 1; + } + } - let mut ret:Vec<&NodeId> = Vec::new(); - for (_, nodes) in found { - for n in nodes { - if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ { - ret.push(n); - } - } - } - ret - } + let mut ret:Vec<&NodeId> = Vec::new(); + for (_, nodes) in found { + for n in nodes { + if ret.len() < BUCKET_SIZE as usize /* && n->endpoint && n->endpoint.isAllowed() */ { + ret.push(n); + } + } + } + ret + } } diff --git a/src/network/host.rs b/src/network/host.rs index 3d8e78e46..1c2a66150 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] //TODO: remove this after everything is done //TODO: remove all unwraps use std::net::{SocketAddr, ToSocketAddrs}; use std::collections::{HashMap}; @@ -16,7 +15,7 @@ use network::handshake::Handshake; use network::session::{Session, SessionData}; use network::{Error, ProtocolHandler}; -const DEFAULT_PORT: u16 = 30304; +const _DEFAULT_PORT: u16 = 30304; const MAX_CONNECTIONS: usize = 1024; const MAX_USER_TIMERS: usize = 32; @@ -54,13 +53,6 @@ pub struct NodeEndpoint { } impl NodeEndpoint { - fn new(address: SocketAddr) -> NodeEndpoint { - NodeEndpoint { - address: address, - address_str: address.to_string(), - udp_port: address.port() - } - } fn from_str(s: &str) -> Result { println!("{:?}", s); let address = s.to_socket_addrs().map(|mut i| i.next()); @@ -87,7 +79,6 @@ struct Node { endpoint: NodeEndpoint, peer_type: PeerType, last_attempted: Option, - confirmed: bool, } impl FromStr for Node { @@ -105,23 +96,10 @@ impl FromStr for Node { endpoint: endpoint, peer_type: PeerType::Optional, last_attempted: None, - confirmed: false }) } } -impl Node { - fn new(id: NodeId, address: SocketAddr, t:PeerType) -> Node { - Node { - id: id, - endpoint: NodeEndpoint::new(address), - peer_type: t, - last_attempted: None, - confirmed: false - } - } -} - impl PartialEq for Node { fn eq(&self, other: &Self) -> bool { self.id == other.id @@ -168,9 +146,9 @@ pub enum HostMessage { pub type UserMessageId = u32; pub struct UserMessage { - protocol: ProtocolId, - id: UserMessageId, - data: Option>, + pub protocol: ProtocolId, + pub id: UserMessageId, + pub data: Option>, } pub type PeerId = usize; @@ -305,14 +283,13 @@ enum ConnectionEntry { pub struct Host { info: HostInfo, - udp_socket: UdpSocket, - listener: TcpListener, + _udp_socket: UdpSocket, + _listener: TcpListener, connections: Slab, timers: Slab, nodes: HashMap, handlers: HashMap>, - idle_timeout: Timeout, - channel: Sender, + _idle_timeout: Timeout, } impl Host { @@ -352,14 +329,13 @@ impl Host { //capabilities: vec![ CapabilityInfo { protocol: "eth".to_string(), version: 63 }], capabilities: Vec::new(), }, - udp_socket: udp_socket, - listener: listener, + _udp_socket: udp_socket, + _listener: listener, connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), nodes: HashMap::new(), handlers: HashMap::new(), - idle_timeout: idle_timeout, - channel: event_loop.channel(), + _idle_timeout: idle_timeout, }; host.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); @@ -639,10 +615,16 @@ impl Handler for Host { NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, USER_TIMER ... LAST_USER_TIMER => { - let protocol = self.timers.get_mut(token).expect("Unknown user timer token").protocol; + let (protocol, delay) = { + let timer = self.timers.get_mut(token).expect("Unknown user timer token"); + (timer.protocol, timer.delay) + }; match self.handlers.get_mut(protocol) { None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(h) => h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()), + Some(h) => { + h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()); + event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); + } } } _ => panic!("Unknown timer token"), diff --git a/src/network/mod.rs b/src/network/mod.rs index c44182430..839624b01 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -90,7 +90,7 @@ pub trait ProtocolHandler: Send { /// Called when a previously connected peer disconnects. fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); /// Timer function called after a timeout created with `HandlerIo::timeout`. - fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken); + fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) -> bool; /// Called when a broadcasted message is received. The message can only be sent from a different protocol handler. fn message(&mut self, io: &mut HandlerIo, message: &Message); } diff --git a/src/network/service.rs b/src/network/service.rs index 986d6dd11..7598ffdd6 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] //TODO: remove this after everything is done - use std::thread::{self, JoinHandle}; use mio::*; use network::{Error, ProtocolHandler}; From 73405cc74c09c788b55dcb11b0c2132f7a1b50e6 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 4 Jan 2016 23:58:59 +0100 Subject: [PATCH 285/379] Fixed timeout signature; Added usage example --- src/network/mod.rs | 47 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/network/mod.rs b/src/network/mod.rs index 839624b01..df0da2c13 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,3 +1,48 @@ +/// Network and general IO module. +/// +/// Example usage for craeting a network service and adding an IO handler: +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::network::*; +/// +/// struct MyHandler; +/// +/// impl ProtocolHandler for MyHandler { +/// fn initialize(&mut self, io: &mut HandlerIo) { +/// io.register_timer(1000); +/// } +/// +/// fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { +/// println!("Received {} ({} bytes) from {}", packet_id, data.len(), peer); +/// } +/// +/// fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { +/// println!("Connected {}", peer); +/// } +/// +/// fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { +/// println!("Disconnected {}", peer); +/// } +/// +/// fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { +/// println!("Timeout {}", timer); +/// } +/// +/// fn message(&mut self, io: &mut HandlerIo, message: &Message) { +/// println!("Message {}:{}", message.protocol, message.id); +/// } +/// } +/// +/// fn main () { +/// let mut service = NetworkService::start().expect("Error creating network service"); +/// service.register_protocol(Box::new(MyHandler), "myproto", &[1u8]); +/// +/// // Wait for quit condition +/// // ... +/// // Drop the service +/// } +/// ``` extern crate mio; mod host; mod connection; @@ -90,7 +135,7 @@ pub trait ProtocolHandler: Send { /// Called when a previously connected peer disconnects. fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); /// Timer function called after a timeout created with `HandlerIo::timeout`. - fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) -> bool; + fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken); /// Called when a broadcasted message is received. The message can only be sent from a different protocol handler. fn message(&mut self, io: &mut HandlerIo, message: &Message); } From 0125125c2764a1c367e66f287191f762e74d161d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 5 Jan 2016 19:05:51 +0100 Subject: [PATCH 286/379] Semantic version test fix. --- src/bytes.rs | 3 +++ src/error.rs | 1 + src/semantic_version.rs | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index d188eb69b..1f1ee8c05 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -92,7 +92,10 @@ pub type Bytes = Vec; /// Slice of bytes to underlying memory pub trait BytesConvertable { + // TODO: rename to as_slice fn bytes(&self) -> &[u8]; + fn as_slice(&self) -> &[u8] { self.bytes() } + fn to_bytes(&self) -> Bytes { self.as_slice().to_vec() } } impl<'a> BytesConvertable for &'a [u8] { diff --git a/src/error.rs b/src/error.rs index 71514d045..329e826bd 100644 --- a/src/error.rs +++ b/src/error.rs @@ -13,6 +13,7 @@ pub enum EthcoreError { FromHex(FromHexError), BaseData(BaseDataError), BadSize, + UnknownName, } impl From for EthcoreError { diff --git a/src/semantic_version.rs b/src/semantic_version.rs index 026902263..4fefaf86a 100644 --- a/src/semantic_version.rs +++ b/src/semantic_version.rs @@ -2,8 +2,8 @@ /// /// # Example /// ``` -/// extern crate ethcore; -/// use ethcore::engine::*; +/// extern crate ethcore_util; +/// use ethcore_util::semantic_version::*; /// fn main() { /// assert_eq!(SemanticVersion::new(1, 2, 3).as_u32(), 0x010203); /// } From 8bf03bb8ff334e0a0b2929c13a35b581f86af7b8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 6 Jan 2016 13:53:23 +0100 Subject: [PATCH 287/379] Secure trie functions & structs. --- src/rlp/tests.rs | 2 +- src/trie/mod.rs | 4 ++++ src/trie/triedbmut.rs | 24 ++++++++++++------------ src/triehash.rs | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs index b4a60a3de..39af862ff 100644 --- a/src/rlp/tests.rs +++ b/src/rlp/tests.rs @@ -4,7 +4,7 @@ use self::json_tests::rlp as rlptest; use std::{fmt, cmp}; use std::str::FromStr; use rlp; -use rlp::{UntrustedRlp, RlpStream, Decodable, View, Stream, Encodable}; +use rlp::{UntrustedRlp, RlpStream, View, Stream}; use uint::U256; #[test] diff --git a/src/trie/mod.rs b/src/trie/mod.rs index 4469926d5..1b5919bd4 100644 --- a/src/trie/mod.rs +++ b/src/trie/mod.rs @@ -4,8 +4,12 @@ pub mod journal; pub mod node; pub mod triedb; pub mod triedbmut; +pub mod sectriedb; +pub mod sectriedbmut; pub use self::trietraits::*; pub use self::standardmap::*; pub use self::triedbmut::*; pub use self::triedb::*; +pub use self::sectriedbmut::*; +pub use self::sectriedb::*; diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index 004912a4c..4c78e7667 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -9,18 +9,6 @@ use super::node::*; use super::journal::*; use super::trietraits::*; -pub struct TrieDBMut<'db> { - db: &'db mut HashDB, - root: &'db mut H256, - pub hash_count: usize, -} - -/// Option-like type allowing either a Node object passthrough or Bytes in the case of data alteration. -enum MaybeChanged<'a> { - Same(Node<'a>), - Changed(Bytes), -} - /// A `Trie` implementation using a generic `HashDB` backing database. /// /// Use it as a `Trie` trait object. You can use `db()` to get the backing database object, `keys` @@ -52,6 +40,18 @@ enum MaybeChanged<'a> { /// assert!(t.db_items_remaining().is_empty()); /// } /// ``` +pub struct TrieDBMut<'db> { + db: &'db mut HashDB, + root: &'db mut H256, + pub hash_count: usize, +} + +/// Option-like type allowing either a Node object passthrough or Bytes in the case of data alteration. +enum MaybeChanged<'a> { + Same(Node<'a>), + Changed(Bytes), +} + impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and empty `root` /// Initialise to the state entailed by the genesis block. diff --git a/src/triehash.rs b/src/triehash.rs index 67ffcc666..74900971c 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -77,6 +77,41 @@ pub fn trie_root(input: Vec<(Vec, Vec)>) -> H256 { gen_trie_root(gen_input) } +/// Generates a key-hashed (secure) trie root hash for a vector of key-values. +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use std::str::FromStr; +/// use util::triehash::*; +/// use util::hash::*; +/// +/// fn main() { +/// let v = vec![ +/// (From::from("doe"), From::from("reindeer")), +/// (From::from("dog"), From::from("puppy")), +/// (From::from("dogglesworth"), From::from("cat")), +/// ]; +/// +/// let root = "d4cd937e4a4368d7931a9cf51686b7e10abb3dce38a39000fd7902a092b64585"; +/// assert_eq!(sec_trie_root(v), H256::from_str(root).unwrap()); +/// } +/// ``` +pub fn sec_trie_root(input: Vec<(Vec, Vec)>) -> H256 { + let gen_input = input + // first put elements into btree to sort them and to remove duplicates + .into_iter() + .fold(BTreeMap::new(), | mut acc, (k, v) | { + acc.insert(k.sha3().to_vec(), v); + acc + }) + // then move them to a vector + .into_iter() + .map(|(k, v)| (as_nibbles(&k), v) ) + .collect(); + + gen_trie_root(gen_input) +} + fn gen_trie_root(input: Vec<(Vec, Vec)>) -> H256 { let mut stream = RlpStream::new(); hash256rlp(&input, 0, &mut stream); From 398231f9a725477288702444b91588f62317764f Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 6 Jan 2016 13:59:48 +0100 Subject: [PATCH 288/379] Additional missing struct stuff. --- src/trie/sectriedb.rs | 59 ++++++++++++++++++++++++++++++++++++ src/trie/sectriedbmut.rs | 65 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/trie/sectriedb.rs create mode 100644 src/trie/sectriedbmut.rs diff --git a/src/trie/sectriedb.rs b/src/trie/sectriedb.rs new file mode 100644 index 000000000..2a8c3b016 --- /dev/null +++ b/src/trie/sectriedb.rs @@ -0,0 +1,59 @@ +use hash::*; +use sha3::*; +use hashdb::*; +use rlp::*; +use super::triedb::*; +use super::trietraits::*; + +/// A `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// +/// Use it as a `Trie` trait object. You can use `raw()` to get the backing TrieDB object. +pub struct SecTrieDB<'db> { + raw: TrieDB<'db> +} + +impl<'db> SecTrieDB<'db> { + /// Create a new trie with the backing database `db` and empty `root` + /// Initialise to the state entailed by the genesis block. + /// This guarantees the trie is built correctly. + pub fn new(db: &'db HashDB, root: &'db H256) -> Self { + SecTrieDB { raw: TrieDB::new(db, root) } + } + + /// Get a reference to the underlying raw TrieDB struct. + pub fn raw(&self) -> &TrieDB { + &self.raw + } + + /// Get a mutable reference to the underlying raw TrieDB struct. + pub fn raw_mut(&mut self) -> &TrieDB { + &mut self.raw + } +} + +impl<'db> Trie for SecTrieDB<'db> { + fn root(&self) -> &H256 { self.raw.root() } + + fn contains(&self, key: &[u8]) -> bool { + self.raw.contains(&key.sha3()) + } + + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.raw.get(&key.sha3()) + } +} + +#[test] +fn trie_to_sectrie() { + use memorydb::*; + use super::triedbmut::*; + + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + { + let mut t = TrieDBMut::new(&mut memdb, &mut root); + t.insert(&(&[0x01u8, 0x23]).sha3(), &[0x01u8, 0x23]); + } + let t = SecTrieDB::new(&memdb, &root); + assert_eq!(t.get(&[0x01u8, 0x23]).unwrap(), &[0x01u8, 0x23]); +} diff --git a/src/trie/sectriedbmut.rs b/src/trie/sectriedbmut.rs new file mode 100644 index 000000000..b0140582d --- /dev/null +++ b/src/trie/sectriedbmut.rs @@ -0,0 +1,65 @@ +use hash::*; +use sha3::*; +use hashdb::*; +use rlp::*; +use super::triedbmut::*; +use super::trietraits::*; + +/// A mutable `Trie` implementation which hashes keys and uses a generic `HashDB` backing database. +/// +/// Use it as a `Trie` or `TrieMut` trait object. You can use `raw()` to get the backing TrieDBMut object. +pub struct SecTrieDBMut<'db> { + raw: TrieDBMut<'db> +} + +impl<'db> SecTrieDBMut<'db> { + /// Create a new trie with the backing database `db` and empty `root` + /// Initialise to the state entailed by the genesis block. + /// This guarantees the trie is built correctly. + pub fn new(db: &'db mut HashDB, root: &'db mut H256) -> Self { + SecTrieDBMut { raw: TrieDBMut::new(db, root) } + } + + /// Create a new trie with the backing database `db` and `root` + /// Panics, if `root` does not exist + pub fn new_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { + SecTrieDBMut { raw: TrieDBMut::new_existing(db, root) } + } +} + +impl<'db> Trie for SecTrieDBMut<'db> { + fn root(&self) -> &H256 { self.raw.root() } + + fn contains(&self, key: &[u8]) -> bool { + self.raw.contains(&key.sha3()) + } + + fn get<'a, 'key>(&'a self, key: &'key [u8]) -> Option<&'a [u8]> where 'a: 'key { + self.raw.get(&key.sha3()) + } +} + +impl<'db> TrieMut for SecTrieDBMut<'db> { + fn insert(&mut self, key: &[u8], value: &[u8]) { + self.raw.insert(&key.sha3(), value); + } + + fn remove(&mut self, key: &[u8]) { + self.raw.remove(&key.sha3()); + } +} + +#[test] +fn sectrie_to_trie() { + use memorydb::*; + use super::triedb::*; + + let mut memdb = MemoryDB::new(); + let mut root = H256::new(); + { + let mut t = SecTrieDBMut::new(&mut memdb, &mut root); + t.insert(&[0x01u8, 0x23], &[0x01u8, 0x23]); + } + let t = TrieDB::new(&memdb, &root); + assert_eq!(t.get(&(&[0x01u8, 0x23]).sha3()).unwrap(), &[0x01u8, 0x23]); +} From 5423797ce4a2e1999f24a3317287ccd152067815 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 6 Jan 2016 14:06:34 +0100 Subject: [PATCH 289/379] Build fix. --- src/semantic_version.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/semantic_version.rs b/src/semantic_version.rs index 368b643e7..92f6ea376 100644 --- a/src/semantic_version.rs +++ b/src/semantic_version.rs @@ -2,14 +2,9 @@ /// /// # Example /// ``` -<<<<<<< HEAD -/// extern crate ethcore_util; -/// use ethcore_util::semantic_version::*; -======= /// extern crate ethcore_util as util; /// use util::semantic_version::*; /// ->>>>>>> master /// fn main() { /// assert_eq!(SemanticVersion::new(1, 2, 3).as_u32(), 0x010203); /// } From 5fc87a334d35112dbc51f3f26a701bf9334ac219 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 7 Jan 2016 23:59:50 +0100 Subject: [PATCH 290/379] Additional work & convenience functions on crypto, bytes & hash. Renaming mut_bytes() to as_slice_mut(). --- src/bytes.rs | 18 ++++++++++++++- src/crypto.rs | 47 ++++++++++++++++++++++++++++++++++------ src/hash.rs | 42 +++++++++++++++++++++++++++++++---- src/rlp/rlpstream.rs | 7 ++++-- src/rlp/rlptraits.rs | 3 +++ src/rlp/untrusted_rlp.rs | 4 ++++ src/sha3.rs | 2 +- src/trie/triedbmut.rs | 2 +- 8 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 1f1ee8c05..0e36a8c8a 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -306,10 +306,26 @@ impl FromBytes for T where T: FixedHash { use std::{mem, ptr}; let mut res: T = mem::uninitialized(); - ptr::copy(bytes.as_ptr(), res.mut_bytes().as_mut_ptr(), T::size()); + ptr::copy(bytes.as_ptr(), res.as_slice_mut().as_mut_ptr(), T::size()); Ok(res) } } } +/// Simple trait to allow for raw population of a Sized object from a byte slice. +pub trait Populatable { + /// Populate self from byte slice `d` in a raw fashion. + fn populate_raw(&mut self, d: &[u8]); +} + +impl Populatable for T where T: Sized { + fn populate_raw(&mut self, d: &[u8]) { + use std::mem; + use std::slice; + use std::io::Write; + unsafe { + slice::from_raw_parts_mut(self as *mut T as *mut u8, mem::size_of::()) + }.write(&d).unwrap(); + } +} diff --git a/src/crypto.rs b/src/crypto.rs index 0d1ae9c05..9a1203921 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -3,9 +3,24 @@ use secp256k1::Secp256k1; use secp256k1::key; use rand::os::OsRng; -pub type Secret=H256; -pub type Public=H512; -pub type Signature=H520; +pub type Secret = H256; +pub type Public = H512; +pub type Signature = H520; + +impl Signature { + /// Create a new signature from the R, S and V componenets. + pub fn from_rsv(r: &H256, s: &H256, v: u8) -> Signature { + use std::ptr; + let mut ret: Signature = Signature::new(); + unsafe { + let retslice: &mut [u8] = &mut ret; + ptr::copy(r.as_ptr(), retslice.as_mut_ptr(), 32); + ptr::copy(s.as_ptr(), retslice.as_mut_ptr().offset(32), 32); + } + ret[64] = v; + ret + } +} #[derive(Debug)] pub enum CryptoError { @@ -92,6 +107,9 @@ impl KeyPair { pub fn secret(&self) -> &Secret { &self.secret } + + /// Sign a message with our secret key. + pub fn sign(&self, message: &H256) -> Result { sign(&self.secret, message) } } /// Recovers Public key from signed message hash. @@ -102,8 +120,11 @@ pub fn recover(signature: &Signature, message: &H256) -> Result Result { use secp256k1::*; @@ -116,6 +137,16 @@ pub fn sign(secret: &Secret, message: &H256) -> Result { signature[64] = rec_id.to_i32() as u8; Ok(signature) } + +/// Check if each component of the signature is in range. +pub fn is_valid(sig: &Signature) -> bool { + sig[64] <= 1 && + H256::from_slice(&sig[0..32]) < h256_from_hex("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141") && + H256::from_slice(&sig[32..64]) < h256_from_hex("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141") && + H256::from_slice(&sig[32..64]) >= h256_from_u64(1) && + H256::from_slice(&sig[0..32]) >= h256_from_u64(1) +} + /// Verify signature. pub fn verify(public: &Public, signature: &Signature, message: &H256) -> Result { use secp256k1::*; @@ -142,6 +173,8 @@ mod tests { use hash::*; use crypto::*; + // TODO: tests for sign/recover roundtrip, at least. + #[test] fn test_signature() { let pair = KeyPair::create().unwrap(); @@ -154,14 +187,14 @@ mod tests { #[test] fn test_invalid_key() { - assert!(KeyPair::from_secret(Secret::from_str("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap()).is_err()); - assert!(KeyPair::from_secret(Secret::from_str("0000000000000000000000000000000000000000000000000000000000000000").unwrap()).is_err()); - assert!(KeyPair::from_secret(Secret::from_str("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141").unwrap()).is_err()); + assert!(KeyPair::from_secret(h256_from_hex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")).is_err()); + assert!(KeyPair::from_secret(h256_from_hex("0000000000000000000000000000000000000000000000000000000000000000")).is_err()); + assert!(KeyPair::from_secret(h256_from_hex("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141")).is_err()); } #[test] fn test_key() { - let pair = KeyPair::from_secret(Secret::from_str("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2").unwrap()).unwrap(); + let pair = KeyPair::from_secret(h256_from_hex("6f7b0d801bc7b5ce7bbd930b84fd0369b3eb25d09be58d64ba811091046f3aa2")).unwrap(); assert_eq!(pair.public().hex(), "101b3ef5a4ea7a1c7928e24c4c75fd053c235d7b80c22ae5c03d145d0ac7396e2a4ffff9adee3133a7b05044a5cee08115fd65145e5165d646bde371010d803c"); } } diff --git a/src/hash.rs b/src/hash.rs index ea041f2ac..e6f9a61c7 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -5,6 +5,7 @@ use std::fmt; use std::ops; use std::hash::{Hash, Hasher}; use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitAnd}; +use std::cmp::{PartialOrd, Ordering}; use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; @@ -21,7 +22,7 @@ pub trait FixedHash: Sized + BytesConvertable { fn random() -> Self; fn randomize(&mut self); fn size() -> usize; - fn mut_bytes(&mut self) -> &mut [u8]; + fn as_slice_mut(&mut self) -> &mut [u8]; fn from_slice(src: &[u8]) -> Self; fn clone_from_slice(&mut self, src: &[u8]) -> usize; fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; @@ -77,7 +78,7 @@ macro_rules! impl_hash { $size } - fn mut_bytes(&mut self) -> &mut [u8] { + fn as_slice_mut(&mut self) -> &mut [u8] { &mut self.0 } @@ -142,7 +143,7 @@ macro_rules! impl_hash { ptr += 1; } index &= mask; - ret.mut_bytes()[m - 1 - index / 8] |= 1 << (index % 8); + ret.as_slice_mut()[m - 1 - index / 8] |= 1 << (index % 8); } ret @@ -208,6 +209,19 @@ macro_rules! impl_hash { } } + impl PartialOrd for $from { + fn partial_cmp(&self, other: &Self) -> Option { + for i in 0..$size { + if self.0[i] > other.0[i] { + return Some(Ordering::Greater); + } else if self.0[i] < other.0[i] { + return Some(Ordering::Less); + } + } + Some(Ordering::Equal) + } + } + impl Hash for $from { fn hash(&self, state: &mut H) where H: Hasher { state.write(&self.0); @@ -311,7 +325,7 @@ macro_rules! impl_hash { } impl<'a> From<&'a U256> for H256 { - fn from(value: &'a U256) -> H256 { + fn from(value: &'a U256) -> H256 { unsafe { let mut ret: H256 = ::std::mem::uninitialized(); value.to_bytes(&mut ret); @@ -340,6 +354,26 @@ impl From
for H256 { } } +pub fn h256_from_hex(s: &str) -> H256 { + use std::str::FromStr; + H256::from_str(s).unwrap() +} + +pub fn h256_from_u64(n: u64) -> H256 { + use uint::U256; + H256::from(&U256::from(n)) +} + +pub fn address_from_hex(s: &str) -> Address { + use std::str::FromStr; + Address::from_str(s).unwrap() +} + +pub fn address_from_u64(n: u64) -> Address { + let h256 = h256_from_u64(n); + From::from(h256) +} + impl_hash!(H32, 4); impl_hash!(H64, 8); impl_hash!(H128, 16); diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 43ae83a9a..3a19cab7d 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -193,8 +193,11 @@ impl Encoder for BasicEncoder { } } - fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () - { + fn emit_raw(&mut self, bytes: &[u8]) -> () { + self.bytes.append_slice(bytes); + } + + fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> () { // get len before inserting a list let before_len = self.bytes.len(); diff --git a/src/rlp/rlptraits.rs b/src/rlp/rlptraits.rs index 067c438bf..5955e132d 100644 --- a/src/rlp/rlptraits.rs +++ b/src/rlp/rlptraits.rs @@ -5,6 +5,8 @@ pub trait Decoder: Sized { where F: FnOnce(&[u8]) -> Result; fn as_list(&self) -> Result, DecoderError>; + + fn as_raw(&self) -> &[u8]; } pub trait Decodable: Sized { @@ -185,6 +187,7 @@ pub trait View<'a, 'view>: Sized { pub trait Encoder { fn emit_value(&mut self, bytes: &[u8]) -> (); fn emit_list(&mut self, f: F) -> () where F: FnOnce(&mut Self) -> (); + fn emit_raw(&mut self, bytes: &[u8]) -> (); } pub trait Encodable { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index a8cecf09f..68956d8d0 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -305,6 +305,10 @@ impl<'a> Decoder for BasicDecoder<'a> { } } + fn as_raw(&self) -> &[u8] { + self.rlp.raw() + } + fn as_list(&self) -> Result, DecoderError> { let v: Vec> = self.rlp.iter() .map(| i | BasicDecoder::new(i)) diff --git a/src/sha3.rs b/src/sha3.rs index b3676720c..e1337c5f7 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -27,7 +27,7 @@ impl Hashable for T where T: BytesConvertable { let mut keccak = Keccak::new_keccak256(); keccak.update(self.bytes()); let mut ret: H256 = uninitialized(); - keccak.finalize(ret.mut_bytes()); + keccak.finalize(ret.as_slice_mut()); ret } } diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index 4c78e7667..e04b7c758 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -686,7 +686,7 @@ mod tests { 0 => encode(&j), _ => { let mut h = H256::new(); - h.mut_bytes()[31] = j as u8; + h.as_slice_mut()[31] = j as u8; encode(&h) }, } From 51ccd5e8fcf9d56414b32ec6738d6976e1982fc1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 00:36:57 +0100 Subject: [PATCH 291/379] Comment TODO. --- src/bytes.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bytes.rs b/src/bytes.rs index 0e36a8c8a..91b0e849c 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -313,6 +313,8 @@ impl FromBytes for T where T: FixedHash { } } +// TODO: tests and additional docs for these two. + /// Simple trait to allow for raw population of a Sized object from a byte slice. pub trait Populatable { /// Populate self from byte slice `d` in a raw fashion. From 806de37cecaed353121f37fb81a867dfdb3dfa01 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 11:02:32 +0100 Subject: [PATCH 292/379] Additional docs. --- src/bytes.rs | 42 +++++++++++++++++++++++++++++++++++++++--- src/crypto.rs | 2 -- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 91b0e849c..e6f71d446 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -35,6 +35,7 @@ //! ``` use std::fmt; +use std::slice; use std::cmp::Ordering; use std::error::Error as StdError; use uint::{U128, U256}; @@ -313,21 +314,56 @@ impl FromBytes for T where T: FixedHash { } } -// TODO: tests and additional docs for these two. - /// Simple trait to allow for raw population of a Sized object from a byte slice. pub trait Populatable { - /// Populate self from byte slice `d` in a raw fashion. + /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. + /// + /// If `d` is smaller, zero-out the remaining bytes. fn populate_raw(&mut self, d: &[u8]); + + /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. + /// + /// If `d` is smaller, will leave some bytes untouched. + fn copy_from_raw(&mut self, d: &[u8]); } impl Populatable for T where T: Sized { fn populate_raw(&mut self, d: &[u8]) { use std::mem; use std::slice; + unsafe { + let mut s = slice::from_raw_parts_mut(self as *mut T as *mut u8, mem::size_of::()); + for i in 0..s.len() { + s[i] = if i < d.len() {d[i]} else {0}; + } + }; + } + + fn copy_from_raw(&mut self, d: &[u8]) { + use std::mem; use std::io::Write; unsafe { slice::from_raw_parts_mut(self as *mut T as *mut u8, mem::size_of::()) }.write(&d).unwrap(); } } + +//impl Populatable for slice {} + +#[test] +fn copy_from_raw() { + let mut x = [255u8; 4]; + x.copy_from_raw(&[1u8; 2][..]); + assert_eq!(x, [1u8, 1, 255, 255]); + x.copy_from_raw(&[1u8; 6][..]); + assert_eq!(x, [1u8, 1, 1, 1]); +} + +#[test] +fn populate_raw() { + let mut x = [255u8; 4]; + x.populate_raw(&[1u8; 2][..]); + assert_eq!(x, [1u8, 1, 0, 0]); + x.populate_raw(&[1u8; 6][..]); + assert_eq!(x, [1u8, 1, 1, 1]); +} \ No newline at end of file diff --git a/src/crypto.rs b/src/crypto.rs index 9a1203921..70b3d0d0b 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -168,8 +168,6 @@ pub fn verify(public: &Public, signature: &Signature, message: &H256) -> Result< #[cfg(test)] mod tests { - - use std::str::FromStr; use hash::*; use crypto::*; From f0da76df7c7ba51f437669063d8853d9c8a39282 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 11:43:11 +0100 Subject: [PATCH 293/379] API cleanups. --- src/bytes.rs | 86 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index e6f71d446..7eae51a44 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -319,43 +319,68 @@ pub trait Populatable { /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. /// /// If `d` is smaller, zero-out the remaining bytes. - fn populate_raw(&mut self, d: &[u8]); + fn populate_raw(&mut self, d: &[u8]) { + let mut s = self.as_slice_mut(); + for i in 0..s.len() { + s[i] = if i < d.len() {d[i]} else {0}; + } + } /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. /// /// If `d` is smaller, will leave some bytes untouched. - fn copy_from_raw(&mut self, d: &[u8]); + fn fax_raw(&mut self, d: &[u8]) { + use std::io::Write; + self.as_slice_mut().write(&d).unwrap(); + } + + /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. + /// + /// If `d` is smaller, will leave some bytes untouched. + fn fax_right_raw(&mut self, d: &[u8]) { + use std::io::Write; + self.as_slice_mut().write(&d).unwrap(); + } + + /// Copies the raw representation of an object `d` to `self`, overwriting as necessary. + /// + /// If `d` is smaller, zero-out the remaining bytes. + fn populate_raw_from(&mut self, d: &BytesConvertable) { self.populate_raw(d.as_slice()); } + + /// Copies the raw representation of an object `d` to `self`, overwriting as necessary. + /// + /// If `d` is smaller, will leave some bytes untouched. + fn fax_raw_from(&mut self, d: &BytesConvertable) { self.populate_raw(d.as_slice()); } + + /// Get the raw slice for this object. + fn as_slice_mut(&mut self) -> &mut [u8]; } impl Populatable for T where T: Sized { - fn populate_raw(&mut self, d: &[u8]) { + fn as_slice_mut(&mut self) -> &mut [u8] { use std::mem; - use std::slice; - unsafe { - let mut s = slice::from_raw_parts_mut(self as *mut T as *mut u8, mem::size_of::()); - for i in 0..s.len() { - s[i] = if i < d.len() {d[i]} else {0}; - } - }; - } - - fn copy_from_raw(&mut self, d: &[u8]) { - use std::mem; - use std::io::Write; unsafe { slice::from_raw_parts_mut(self as *mut T as *mut u8, mem::size_of::()) - }.write(&d).unwrap(); + } } } -//impl Populatable for slice {} +impl Populatable for [T] where T: Sized { + fn as_slice_mut(&mut self) -> &mut [u8] { + use std::mem; + unsafe { + slice::from_raw_parts_mut(self.as_mut_ptr() as *mut u8, mem::size_of::() * self.len()) + } + } +} #[test] -fn copy_from_raw() { +fn fax_raw() { let mut x = [255u8; 4]; - x.copy_from_raw(&[1u8; 2][..]); + x.fax_raw(&[1u8; 2][..]); assert_eq!(x, [1u8, 1, 255, 255]); - x.copy_from_raw(&[1u8; 6][..]); + let mut x = [255u8; 4]; + x.fax_raw(&[1u8; 6][..]); assert_eq!(x, [1u8, 1, 1, 1]); } @@ -364,6 +389,27 @@ fn populate_raw() { let mut x = [255u8; 4]; x.populate_raw(&[1u8; 2][..]); assert_eq!(x, [1u8, 1, 0, 0]); + let mut x = [255u8; 4]; x.populate_raw(&[1u8; 6][..]); assert_eq!(x, [1u8, 1, 1, 1]); +} + +#[test] +fn populate_raw_dyn() { + let mut x = [255u8; 4]; + x.populate_raw(&[1u8; 2][..]); + assert_eq!(&x[..], [1u8, 1, 0, 0]); + let mut x = [255u8; 4]; + x.populate_raw(&[1u8; 6][..]); + assert_eq!(&x[..], [1u8, 1, 1, 1]); +} + +#[test] +fn fax_raw_dyn() { + let mut x = [255u8; 4]; + x.fax_raw(&[1u8; 2][..]); + assert_eq!(&x[..], [1u8, 1, 255, 255]); + let mut x = [255u8; 4]; + x.fax_raw(&[1u8; 6][..]); + assert_eq!(&x[..], [1u8, 1, 1, 1]); } \ No newline at end of file From 3018c4a28a68072ff71b6229c7ddb491b1a541f6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 11:52:46 +0100 Subject: [PATCH 294/379] Additional Populatable API tweaks. --- src/hash.rs | 9 ++------- src/sha3.rs | 4 ++-- src/trie/triedbmut.rs | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index e6f9a61c7..2fb4b71e6 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -10,19 +10,18 @@ use rustc_serialize::hex::*; use error::EthcoreError; use rand::Rng; use rand::os::OsRng; -use bytes::BytesConvertable; +use bytes::{BytesConvertable,Populatable}; use math::log2; use uint::U256; /// Trait for a fixed-size byte array to be used as the output of hash functions. /// /// Note: types implementing `FixedHash` must be also `BytesConvertable`. -pub trait FixedHash: Sized + BytesConvertable { +pub trait FixedHash: Sized + BytesConvertable + Populatable { fn new() -> Self; fn random() -> Self; fn randomize(&mut self); fn size() -> usize; - fn as_slice_mut(&mut self) -> &mut [u8]; fn from_slice(src: &[u8]) -> Self; fn clone_from_slice(&mut self, src: &[u8]) -> usize; fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; @@ -78,10 +77,6 @@ macro_rules! impl_hash { $size } - fn as_slice_mut(&mut self) -> &mut [u8] { - &mut self.0 - } - // TODO: remove once slice::clone_from_slice is stable #[inline] fn clone_from_slice(&mut self, src: &[u8]) -> usize { diff --git a/src/sha3.rs b/src/sha3.rs index e1337c5f7..f3faa4e3b 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -2,8 +2,8 @@ use std::mem::uninitialized; use tiny_keccak::Keccak; -use bytes::BytesConvertable; -use hash::{FixedHash, H256}; +use bytes::{BytesConvertable,Populatable}; +use hash::H256; /// Types implementing this trait are sha3able. /// diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index e04b7c758..a51e56a83 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -668,7 +668,7 @@ mod tests { use env_logger; use rand::random; use std::collections::HashSet; - use bytes::{ToPretty,Bytes}; + use bytes::{ToPretty,Bytes,Populatable}; use super::super::node::*; use super::super::trietraits::*; From f81fb2de51dcd710e7dc6ec9757eb810d939604e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 12:02:26 +0100 Subject: [PATCH 295/379] Fix and some tests for opaque types in populatable. --- src/bytes.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 7eae51a44..133de416b 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -334,14 +334,6 @@ pub trait Populatable { self.as_slice_mut().write(&d).unwrap(); } - /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. - /// - /// If `d` is smaller, will leave some bytes untouched. - fn fax_right_raw(&mut self, d: &[u8]) { - use std::io::Write; - self.as_slice_mut().write(&d).unwrap(); - } - /// Copies the raw representation of an object `d` to `self`, overwriting as necessary. /// /// If `d` is smaller, zero-out the remaining bytes. @@ -350,7 +342,7 @@ pub trait Populatable { /// Copies the raw representation of an object `d` to `self`, overwriting as necessary. /// /// If `d` is smaller, will leave some bytes untouched. - fn fax_raw_from(&mut self, d: &BytesConvertable) { self.populate_raw(d.as_slice()); } + fn fax_raw_from(&mut self, d: &BytesConvertable) { self.fax_raw(d.as_slice()); } /// Get the raw slice for this object. fn as_slice_mut(&mut self) -> &mut [u8]; @@ -412,4 +404,16 @@ fn fax_raw_dyn() { let mut x = [255u8; 4]; x.fax_raw(&[1u8; 6][..]); assert_eq!(&x[..], [1u8, 1, 1, 1]); +} + +#[test] +fn populate_big_types() { + use hash::*; + let a = address_from_hex("ffffffffffffffffffffffffffffffffffffffff"); + let mut h = h256_from_u64(0x69); + h.populate_raw_from(&a); + assert_eq!(h, h256_from_hex("ffffffffffffffffffffffffffffffffffffffff000000000000000000000000")); + let mut h = h256_from_u64(0x69); + h.fax_raw_from(&a); + assert_eq!(h, h256_from_hex("ffffffffffffffffffffffffffffffffffffffff000000000000000000000069")); } \ No newline at end of file From e4ed6467a92354e739f2d1b3aabfd6d4562a7d11 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 12:05:38 +0100 Subject: [PATCH 296/379] Fixes and renaming fax->copy. --- src/bytes.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 133de416b..5ffc72c08 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -329,7 +329,7 @@ pub trait Populatable { /// Copies a bunch of bytes `d` to `self`, overwriting as necessary. /// /// If `d` is smaller, will leave some bytes untouched. - fn fax_raw(&mut self, d: &[u8]) { + fn copy_raw(&mut self, d: &[u8]) { use std::io::Write; self.as_slice_mut().write(&d).unwrap(); } @@ -342,7 +342,7 @@ pub trait Populatable { /// Copies the raw representation of an object `d` to `self`, overwriting as necessary. /// /// If `d` is smaller, will leave some bytes untouched. - fn fax_raw_from(&mut self, d: &BytesConvertable) { self.fax_raw(d.as_slice()); } + fn copy_raw_from(&mut self, d: &BytesConvertable) { self.copy_raw(d.as_slice()); } /// Get the raw slice for this object. fn as_slice_mut(&mut self) -> &mut [u8]; @@ -369,10 +369,10 @@ impl Populatable for [T] where T: Sized { #[test] fn fax_raw() { let mut x = [255u8; 4]; - x.fax_raw(&[1u8; 2][..]); + x.copy_raw(&[1u8; 2][..]); assert_eq!(x, [1u8, 1, 255, 255]); let mut x = [255u8; 4]; - x.fax_raw(&[1u8; 6][..]); + x.copy_raw(&[1u8; 6][..]); assert_eq!(x, [1u8, 1, 1, 1]); } @@ -399,10 +399,10 @@ fn populate_raw_dyn() { #[test] fn fax_raw_dyn() { let mut x = [255u8; 4]; - x.fax_raw(&[1u8; 2][..]); + x.copy_raw(&[1u8; 2][..]); assert_eq!(&x[..], [1u8, 1, 255, 255]); let mut x = [255u8; 4]; - x.fax_raw(&[1u8; 6][..]); + x.copy_raw(&[1u8; 6][..]); assert_eq!(&x[..], [1u8, 1, 1, 1]); } @@ -414,6 +414,6 @@ fn populate_big_types() { h.populate_raw_from(&a); assert_eq!(h, h256_from_hex("ffffffffffffffffffffffffffffffffffffffff000000000000000000000000")); let mut h = h256_from_u64(0x69); - h.fax_raw_from(&a); + h.copy_raw_from(&a); assert_eq!(h, h256_from_hex("ffffffffffffffffffffffffffffffffffffffff000000000000000000000069")); } \ No newline at end of file From 288794fd275d0784323e5edc3fa40b9381afbf7e Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 13:10:00 +0100 Subject: [PATCH 297/379] Style --- src/network/connection.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/network/connection.rs b/src/network/connection.rs index 9fbd4df76..b3f655ffd 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -72,7 +72,7 @@ impl Connection { } } - pub fn send(&mut self, data: Bytes) { //TODO: take ownership version + pub fn send(&mut self, data: Bytes) { if data.len() != 0 { self.send_queue.push_back(Cursor::new(data)); } @@ -106,7 +106,7 @@ impl Connection { }.and_then(|r| { if r == WriteStatus::Complete { self.send_queue.pop_front(); - }; + } if self.send_queue.is_empty() { self.interest.remove(EventSet::writable()); } @@ -121,7 +121,7 @@ impl Connection { trace!(target: "net", "connection register; token={:?}", self.token); self.interest.insert(EventSet::readable()); event_loop.register(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { - error!("Failed to reregister {:?}, {:?}", self.token, e); + error!("Failed to register {:?}, {:?}", self.token, e); Err(e) }) } @@ -210,7 +210,6 @@ impl EncryptedConnection { } pub fn send_packet(&mut self, payload: &[u8]) -> Result<(), Error> { - println!("HEADER"); let mut header = RlpStream::new(); let len = payload.len() as usize; header.append_raw(&[(len >> 16) as u8, (len >> 8) as u8, len as u8], 1); From e6623c0edd4130436e74c93dad343af1ae9794ab Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 13:33:38 +0100 Subject: [PATCH 298/379] Removed wrong file --- src/key_manager.rs | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 src/key_manager.rs diff --git a/src/key_manager.rs b/src/key_manager.rs deleted file mode 100644 index a14677867..000000000 --- a/src/key_manager.rs +++ /dev/null @@ -1,8 +0,0 @@ - -enum Kdf { - Pbkdf2Sha256, - Scrypt -} - - - From 396a1755e3373376b752665a2ecc3a36d473a557 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 13:49:00 +0100 Subject: [PATCH 299/379] More style --- src/crypto.rs | 2 - src/network/connection.rs | 14 +- src/network/handshake.rs | 6 +- src/network/host.rs | 307 ++++++++++++++++++-------------------- 4 files changed, 154 insertions(+), 175 deletions(-) diff --git a/src/crypto.rs b/src/crypto.rs index f485351e9..a426cf9ba 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -313,8 +313,6 @@ pub mod aes { #[cfg(test)] mod tests { - - use std::str::FromStr; use hash::*; use crypto::*; diff --git a/src/network/connection.rs b/src/network/connection.rs index b3f655ffd..2a8025b83 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -1,4 +1,3 @@ -#![allow(dead_code)] //TODO: remove this after everything is done use std::collections::VecDeque; use mio::{Token, EventSet, EventLoop, Timeout, PollOpt, TryRead, TryWrite}; use mio::tcp::*; @@ -183,7 +182,6 @@ impl EncryptedConnection { key_material.sha3().copy_to(&mut key_material[32..64]); let mac_encoder = EcbEncryptor::new(AesSafe256Encryptor::new(&key_material[32..64]), NoPadding); - println!("SESSION key: {}", H256::from_slice(&key_material[32..64]).hex()); let mut egress_mac = Keccak::new_keccak256(); let mut mac_material = &H256::from_slice(&key_material[32..64]) ^ &handshake.remote_nonce; @@ -292,17 +290,10 @@ impl EncryptedConnection { let mut prev = H128::new(); mac.clone().finalize(&mut prev); let mut enc = H128::new(); - println!("before: {}", prev.hex()); mac_encoder.encrypt(&mut RefReadBuffer::new(&prev), &mut RefWriteBuffer::new(&mut enc), true).unwrap(); mac_encoder.reset(); - println!("after {}", enc.hex()); - if !seed.is_empty() { - enc = enc ^ H128::from_slice(seed); - } - else { - enc = enc ^ prev; - } + enc = enc ^ if seed.is_empty() { prev } else { H128::from_slice(seed) }; mac.update(&enc); } @@ -349,7 +340,7 @@ impl EncryptedConnection { } #[test] -pub fn ctest() { +pub fn test_encryption() { use hash::*; use std::str::FromStr; let key = H256::from_str("2212767d793a7a3d66f869ae324dd11bd17044b82c9f463b8a541a4d089efec5").unwrap(); @@ -363,7 +354,6 @@ pub fn ctest() { let mut encoder = EcbEncryptor::new(AesSafe256Encryptor::new(&key), NoPadding); encoder.encrypt(&mut RefReadBuffer::new(&before), &mut RefWriteBuffer::new(&mut got), true).unwrap(); encoder.reset(); - println!("got: {} ", got.hex()); assert_eq!(got, after); got = H128::new(); encoder.encrypt(&mut RefReadBuffer::new(&before2), &mut RefWriteBuffer::new(&mut got), true).unwrap(); diff --git a/src/network/handshake.rs b/src/network/handshake.rs index f25d6ef89..9f4ca4d97 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -30,8 +30,8 @@ pub struct Handshake { pub ack_cipher: Bytes } -const AUTH_PACKET_SIZE:usize = 307; -const ACK_PACKET_SIZE:usize = 210; +const AUTH_PACKET_SIZE: usize = 307; +const ACK_PACKET_SIZE: usize = 210; impl Handshake { pub fn new(token: Token, id: &NodeId, socket: TcpStream, nonce: &H256) -> Result { @@ -106,7 +106,7 @@ impl Handshake { pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); - self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); + self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); try!(self.connection.register(event_loop)); Ok(()) } diff --git a/src/network/host.rs b/src/network/host.rs index 1c2a66150..e41480c23 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -1,4 +1,3 @@ -//TODO: remove all unwraps use std::net::{SocketAddr, ToSocketAddrs}; use std::collections::{HashMap}; use std::hash::{Hash, Hasher}; @@ -19,42 +18,41 @@ const _DEFAULT_PORT: u16 = 30304; const MAX_CONNECTIONS: usize = 1024; const MAX_USER_TIMERS: usize = 32; -const IDEAL_PEERS:u32 = 10; +const IDEAL_PEERS: u32 = 10; pub type NodeId = H512; pub type TimerToken = usize; #[derive(Debug)] struct NetworkConfiguration { - listen_address: SocketAddr, - public_address: SocketAddr, - no_nat: bool, - no_discovery: bool, - pin: bool, + listen_address: SocketAddr, + public_address: SocketAddr, + nat_enabled: bool, + discovery_enabled: bool, + pin: bool, } impl NetworkConfiguration { - fn new() -> NetworkConfiguration { - NetworkConfiguration { - listen_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), - public_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), - no_nat: false, - no_discovery: false, - pin: false, - } - } + fn new() -> NetworkConfiguration { + NetworkConfiguration { + listen_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), + public_address: SocketAddr::from_str("0.0.0.0:30304").unwrap(), + nat_enabled: true, + discovery_enabled: true, + pin: false, + } + } } #[derive(Debug)] pub struct NodeEndpoint { - address: SocketAddr, - address_str: String, - udp_port: u16 + address: SocketAddr, + address_str: String, + udp_port: u16 } impl NodeEndpoint { - fn from_str(s: &str) -> Result { - println!("{:?}", s); + fn from_str(s: &str) -> Result { let address = s.to_socket_addrs().map(|mut i| i.next()); match address { Ok(Some(a)) => Ok(NodeEndpoint { @@ -65,19 +63,19 @@ impl NodeEndpoint { Ok(_) => Err(Error::AddressResolve(None)), Err(e) => Err(Error::AddressResolve(Some(e))) } - } + } } #[derive(PartialEq, Eq, Copy, Clone)] enum PeerType { - Required, - Optional + Required, + Optional } struct Node { - id: NodeId, - endpoint: NodeEndpoint, - peer_type: PeerType, + id: NodeId, + endpoint: NodeEndpoint, + peer_type: PeerType, last_attempted: Option, } @@ -91,26 +89,26 @@ impl FromStr for Node { (NodeId::new(), try!(NodeEndpoint::from_str(s))) }; - Ok(Node { - id: id, - endpoint: endpoint, - peer_type: PeerType::Optional, + Ok(Node { + id: id, + endpoint: endpoint, + peer_type: PeerType::Optional, last_attempted: None, - }) + }) } } impl PartialEq for Node { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } } impl Eq for Node { } impl Hash for Node { - fn hash(&self, state: &mut H) where H: Hasher { - self.id.hash(state) - } + fn hash(&self, state: &mut H) where H: Hasher { + self.id.hash(state) + } } // Tokens @@ -128,7 +126,7 @@ pub type PacketId = u8; pub type ProtocolId = &'static str; pub enum HostMessage { - Shutdown, + Shutdown, AddHandler { handler: Box, protocol: ProtocolId, @@ -172,8 +170,8 @@ impl Encodable for CapabilityInfo { /// IO access point pub struct HostIo<'s> { protocol: ProtocolId, - connections: &'s mut Slab, - timers: &'s mut Slab, + connections: &'s mut Slab, + timers: &'s mut Slab, session: Option, event_loop: &'s mut EventLoop, } @@ -217,9 +215,9 @@ impl<'s> HostIo<'s> { /// Register a new IO timer. Returns a new timer toke. 'ProtocolHandler::timeout' will be called with the token. pub fn register_timer(&mut self, ms: u64) -> Result{ match self.timers.insert(UserTimer { - delay: ms, - protocol: self.protocol, - }) { + delay: ms, + protocol: self.protocol, + }) { Ok(token) => { self.event_loop.timeout_ms(token, ms).expect("Error registering user timer"); Ok(token.as_usize()) @@ -253,8 +251,8 @@ struct UserTimer { } pub struct HostInfo { - keys: KeyPair, - config: NetworkConfiguration, + keys: KeyPair, + config: NetworkConfiguration, nonce: H256, pub protocol_version: u32, pub client_version: String, @@ -283,42 +281,38 @@ enum ConnectionEntry { pub struct Host { info: HostInfo, - _udp_socket: UdpSocket, - _listener: TcpListener, - connections: Slab, - timers: Slab, + _udp_socket: UdpSocket, + _listener: TcpListener, + connections: Slab, + timers: Slab, nodes: HashMap, handlers: HashMap>, _idle_timeout: Timeout, } impl Host { - pub fn start(event_loop: &mut EventLoop) -> Result<(), Error> { - let config = NetworkConfiguration::new(); + pub fn start(event_loop: &mut EventLoop) -> Result<(), Error> { + let config = NetworkConfiguration::new(); /* - match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { - Some(iface) => config.public_address = iface.addr.unwrap(), - None => warn!("No public network interface"), - } - */ + match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { + Some(iface) => config.public_address = iface.addr.unwrap(), + None => warn!("No public network interface"), + } + */ - let addr = config.listen_address; - // Setup the server socket - let listener = TcpListener::bind(&addr).unwrap(); - // Start listening for incoming connections - event_loop.register(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); - // Setup the client socket - //let sock = TcpStream::connect(&addr).unwrap(); - // Register the socket - //self.event_loop.register(&sock, CLIENT, EventSet::readable(), PollOpt::edge()).unwrap(); - let idle_timeout = event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay - // open the udp socket - let udp_socket = UdpSocket::bound(&addr).unwrap(); - event_loop.register(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); - event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); + let addr = config.listen_address; + // Setup the server socket + let listener = TcpListener::bind(&addr).unwrap(); + // Start listening for incoming connections + event_loop.register(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); + let idle_timeout = event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay + // open the udp socket + let udp_socket = UdpSocket::bound(&addr).unwrap(); + event_loop.register(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); + event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); let port = config.listen_address.port(); - let mut host = Host { + let mut host = Host { info: HostInfo { keys: KeyPair::create().unwrap(), config: config, @@ -326,17 +320,16 @@ impl Host { protocol_version: 4, client_version: "parity".to_string(), listen_port: port, - //capabilities: vec![ CapabilityInfo { protocol: "eth".to_string(), version: 63 }], capabilities: Vec::new(), }, - _udp_socket: udp_socket, - _listener: listener, + _udp_socket: udp_socket, + _listener: listener, connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), nodes: HashMap::new(), handlers: HashMap::new(), _idle_timeout: idle_timeout, - }; + }; host.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); @@ -345,9 +338,9 @@ impl Host { host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); - try!(event_loop.run(&mut host)); + try!(event_loop.run(&mut host)); Ok(()) - } + } fn add_node(&mut self, id: &str) { match Node::from_str(id) { @@ -358,9 +351,9 @@ impl Host { } } - fn maintain_network(&mut self, event_loop: &mut EventLoop) { - self.connect_peers(event_loop); - } + fn maintain_network(&mut self, event_loop: &mut EventLoop) { + self.connect_peers(event_loop); + } fn have_session(&self, id: &NodeId) -> bool { self.connections.iter().any(|e| match e { &ConnectionEntry::Session(ref s) => s.info.id.eq(&id), _ => false }) @@ -370,7 +363,7 @@ impl Host { self.connections.iter().any(|e| match e { &ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) } - fn connect_peers(&mut self, event_loop: &mut EventLoop) { + fn connect_peers(&mut self, event_loop: &mut EventLoop) { struct NodeInfo { id: NodeId, @@ -416,7 +409,7 @@ impl Host { } } } - } + } fn connect_peer(&mut self, id: &NodeId, event_loop: &mut EventLoop) { if self.have_session(id) @@ -434,8 +427,6 @@ impl Host { let node = self.nodes.get_mut(id).unwrap(); node.last_attempted = Some(::time::now()); - - //blog(NetConnect) << "Attempting connection to node" << _p->id << "@" << ep << "from" << id(); match TcpStream::connect(&node.endpoint.address) { Ok(socket) => socket, Err(_) => { @@ -573,7 +564,7 @@ impl Host { debug!(target: "net", "Session construction error: {:?}", e); None }), - _ => { panic!("No handshake to create a session from"); } + _ => { panic!("No handshake to create a session from"); } } }).expect("Error updating slab with session"); } @@ -584,90 +575,90 @@ impl Host { fn kill_connection(&mut self, token: Token, _event_loop: &mut EventLoop) { self.connections.remove(token); } -} + } -impl Handler for Host { - type Timeout = Token; - type Message = HostMessage; + impl Handler for Host { + type Timeout = Token; + type Message = HostMessage; - fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { - if events.is_readable() { + fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { + if events.is_readable() { + match token.as_usize() { + TCP_ACCEPT => self.accept(event_loop), + IDLE => self.maintain_network(event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), + NODETABLE_RECEIVE => {}, + _ => panic!("Received unknown readable token"), + } + } + else if events.is_writable() { + match token.as_usize() { + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), + _ => panic!("Received unknown writable token"), + } + } + } + + fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { match token.as_usize() { - TCP_ACCEPT => self.accept(event_loop), IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), - NODETABLE_RECEIVE => {}, - _ => panic!("Received unknown readable token"), - } - } - else if events.is_writable() { - match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), - _ => panic!("Received unknown writable token"), - } - } - } - - fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { - match token.as_usize() { - IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), - NODETABLE_DISCOVERY => {}, - NODETABLE_MAINTAIN => {}, - USER_TIMER ... LAST_USER_TIMER => { - let (protocol, delay) = { - let timer = self.timers.get_mut(token).expect("Unknown user timer token"); - (timer.protocol, timer.delay) - }; - match self.handlers.get_mut(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(h) => { - h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()); - event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), + NODETABLE_DISCOVERY => {}, + NODETABLE_MAINTAIN => {}, + USER_TIMER ... LAST_USER_TIMER => { + let (protocol, delay) = { + let timer = self.timers.get_mut(token).expect("Unknown user timer token"); + (timer.protocol, timer.delay) + }; + match self.handlers.get_mut(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(h) => { + h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()); + event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); + } } } + _ => panic!("Unknown timer token"), } - _ => panic!("Unknown timer token"), } - } - fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { - match msg { - HostMessage::Shutdown => event_loop.shutdown(), - HostMessage::AddHandler { - handler, - protocol, - versions - } => { - self.handlers.insert(protocol, handler); - for v in versions { - self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); - } - }, - HostMessage::Send { - peer, - packet_id, - protocol, - data, - } => { - match self.connections.get_mut(Token(peer as usize)) { - Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { - warn!(target: "net", "Send error: {:?}", e); - }); //TODO: don't copy vector data - }, - _ => { - warn!(target: "net", "Send: Peer does not exist"); + fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { + match msg { + HostMessage::Shutdown => event_loop.shutdown(), + HostMessage::AddHandler { + handler, + protocol, + versions + } => { + self.handlers.insert(protocol, handler); + for v in versions { + self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); } - } - }, - HostMessage::UserMessage(message) => { - for (p, h) in self.handlers.iter_mut() { - if p != &message.protocol { - h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); + }, + HostMessage::Send { + peer, + packet_id, + protocol, + data, + } => { + match self.connections.get_mut(Token(peer as usize)) { + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { + warn!(target: "net", "Send error: {:?}", e); + }); //TODO: don't copy vector data + }, + _ => { + warn!(target: "net", "Send: Peer does not exist"); + } + } + }, + HostMessage::UserMessage(message) => { + for (p, h) in self.handlers.iter_mut() { + if p != &message.protocol { + h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); + } } } } } } -} From 469b0a17dc037d680e48781a4c96fc5853ecabac Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 13:55:44 +0100 Subject: [PATCH 300/379] More style --- src/network/host.rs | 257 ++++++++++++++++++++++---------------------- 1 file changed, 127 insertions(+), 130 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index e41480c23..e8040a10e 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -462,25 +462,23 @@ impl Host { fn connection_writable(&mut self, token: Token, event_loop: &mut EventLoop) { let mut kill = false; let mut create_session = false; - { - match self.connections.get_mut(token) { - Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.writable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Handshake write error: {:?}", e); - kill = true; - }); - create_session = h.done(); - }, - Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.writable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Session write error: {:?}", e); - kill = true; - }); - } - _ => { - warn!(target: "net", "Received event for unknown connection"); - } - }; + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Handshake(ref mut h)) => { + h.writable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Handshake write error: {:?}", e); + kill = true; + }); + create_session = h.done(); + }, + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.writable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Session write error: {:?}", e); + kill = true; + }); + } + _ => { + warn!(target: "net", "Received event for unknown connection"); + } } if kill { self.kill_connection(token, event_loop); @@ -496,46 +494,44 @@ impl Host { let mut create_session = false; let mut ready_data: Vec = Vec::new(); let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; - { - match self.connections.get_mut(token) { - Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.readable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Handshake read error: {:?}", e); - kill = true; - }); - create_session = h.done(); - }, - Some(&mut ConnectionEntry::Session(ref mut s)) => { - let sd = { s.readable(event_loop, &self.info).unwrap_or_else(|e| { - debug!(target: "net", "Session read error: {:?}", e); - kill = true; - SessionData::None - }) }; - match sd { - SessionData::Ready => { - for (p, _) in self.handlers.iter_mut() { - if s.have_capability(p) { - ready_data.push(p); - } + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Handshake(ref mut h)) => { + h.readable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Handshake read error: {:?}", e); + kill = true; + }); + create_session = h.done(); + }, + Some(&mut ConnectionEntry::Session(ref mut s)) => { + let sd = { s.readable(event_loop, &self.info).unwrap_or_else(|e| { + debug!(target: "net", "Session read error: {:?}", e); + kill = true; + SessionData::None + }) }; + match sd { + SessionData::Ready => { + for (p, _) in self.handlers.iter_mut() { + if s.have_capability(p) { + ready_data.push(p); } - }, - SessionData::Packet { - data, - protocol, - packet_id, - } => { - match self.handlers.get_mut(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(_) => packet_data = Some((protocol, packet_id, data)), - } - }, - SessionData::None => {}, - } + } + }, + SessionData::Packet { + data, + protocol, + packet_id, + } => { + match self.handlers.get_mut(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(_) => packet_data = Some((protocol, packet_id, data)), + } + }, + SessionData::None => {}, } - _ => { - warn!(target: "net", "Received event for unknown connection"); - } - }; + } + _ => { + warn!(target: "net", "Received event for unknown connection"); + } } if kill { self.kill_connection(token, event_loop); @@ -572,93 +568,94 @@ impl Host { fn connection_timeout(&mut self, token: Token, event_loop: &mut EventLoop) { self.kill_connection(token, event_loop) } + fn kill_connection(&mut self, token: Token, _event_loop: &mut EventLoop) { self.connections.remove(token); } - } +} - impl Handler for Host { - type Timeout = Token; - type Message = HostMessage; +impl Handler for Host { + type Timeout = Token; + type Message = HostMessage; - fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { - if events.is_readable() { - match token.as_usize() { - TCP_ACCEPT => self.accept(event_loop), - IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), - NODETABLE_RECEIVE => {}, - _ => panic!("Received unknown readable token"), - } - } - else if events.is_writable() { - match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), - _ => panic!("Received unknown writable token"), - } - } - } - - fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { + fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { + if events.is_readable() { match token.as_usize() { + TCP_ACCEPT => self.accept(event_loop), IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), - NODETABLE_DISCOVERY => {}, - NODETABLE_MAINTAIN => {}, - USER_TIMER ... LAST_USER_TIMER => { - let (protocol, delay) = { - let timer = self.timers.get_mut(token).expect("Unknown user timer token"); - (timer.protocol, timer.delay) - }; - match self.handlers.get_mut(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(h) => { - h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()); - event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); - } - } - } - _ => panic!("Unknown timer token"), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), + NODETABLE_RECEIVE => {}, + _ => panic!("Received unknown readable token"), } } + else if events.is_writable() { + match token.as_usize() { + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), + _ => panic!("Received unknown writable token"), + } + } + } - fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { - match msg { - HostMessage::Shutdown => event_loop.shutdown(), - HostMessage::AddHandler { - handler, - protocol, - versions - } => { - self.handlers.insert(protocol, handler); - for v in versions { - self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); + fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { + match token.as_usize() { + IDLE => self.maintain_network(event_loop), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), + NODETABLE_DISCOVERY => {}, + NODETABLE_MAINTAIN => {}, + USER_TIMER ... LAST_USER_TIMER => { + let (protocol, delay) = { + let timer = self.timers.get_mut(token).expect("Unknown user timer token"); + (timer.protocol, timer.delay) + }; + match self.handlers.get_mut(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(h) => { + h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()); + event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); } - }, - HostMessage::Send { - peer, - packet_id, - protocol, - data, - } => { - match self.connections.get_mut(Token(peer as usize)) { - Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { - warn!(target: "net", "Send error: {:?}", e); - }); //TODO: don't copy vector data - }, - _ => { - warn!(target: "net", "Send: Peer does not exist"); - } + } + } + _ => panic!("Unknown timer token"), + } + } + + fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { + match msg { + HostMessage::Shutdown => event_loop.shutdown(), + HostMessage::AddHandler { + handler, + protocol, + versions + } => { + self.handlers.insert(protocol, handler); + for v in versions { + self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); + } + }, + HostMessage::Send { + peer, + packet_id, + protocol, + data, + } => { + match self.connections.get_mut(Token(peer as usize)) { + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { + warn!(target: "net", "Send error: {:?}", e); + }); //TODO: don't copy vector data + }, + _ => { + warn!(target: "net", "Send: Peer does not exist"); } - }, - HostMessage::UserMessage(message) => { - for (p, h) in self.handlers.iter_mut() { - if p != &message.protocol { - h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); - } + } + }, + HostMessage::UserMessage(message) => { + for (p, h) in self.handlers.iter_mut() { + if p != &message.protocol { + h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); } } } } } +} From 9286a03656e7d65fde096e7b608fa1945d9edefd Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 15:52:43 +0100 Subject: [PATCH 301/379] Even more style --- src/network/session.rs | 9 +++------ src/rlp/rlp.rs | 4 ++-- src/rlp/rlptraits.rs | 4 ++-- src/rlp/tests.rs | 12 ++++++------ src/rlp/untrusted_rlp.rs | 6 +++--- src/trie/node.rs | 4 ++-- src/trie/triedbmut.rs | 20 ++++++++++---------- src/uint.rs | 26 ++++++++++++++++++++++++-- 8 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/network/session.rs b/src/network/session.rs index a34f6ff1a..720902150 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -1,5 +1,3 @@ -//#![allow(dead_code)] //TODO: remove this after everything is done - use mio::*; use hash::*; use rlp::*; @@ -126,15 +124,15 @@ impl Session { } match packet_id { PACKET_HELLO => { - let rlp = UntrustedRlp::new(&packet.data[1..]); //TODO: validate rlp expected size + let rlp = UntrustedRlp::new(&packet.data[1..]); //TODO: validate rlp expected size try!(self.read_hello(&rlp, host)); Ok(SessionData::Ready) - } + }, PACKET_DISCONNECT => Err(Error::Disconnect(DisconnectReason::DisconnectRequested)), PACKET_PING => { try!(self.write_pong()); Ok(SessionData::None) - } + }, PACKET_GET_PEERS => Ok(SessionData::None), //TODO; PACKET_PEERS => Ok(SessionData::None), PACKET_USER ... PACKET_LAST => { @@ -226,7 +224,6 @@ impl Session { self.send(try!(Session::prepare(PACKET_PONG, 0))) } - fn disconnect(&mut self, reason: DisconnectReason) -> Error { let mut rlp = RlpStream::new(); rlp.append(&(PACKET_DISCONNECT as u32)); diff --git a/src/rlp/rlp.rs b/src/rlp/rlp.rs index ac830cc9c..2179643d1 100644 --- a/src/rlp/rlp.rs +++ b/src/rlp/rlp.rs @@ -29,8 +29,8 @@ impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { } } - fn raw(&'view self) -> &'a [u8] { - self.rlp.raw() + fn as_raw(&'view self) -> &'a [u8] { + self.rlp.as_raw() } fn prototype(&self) -> Self::Prototype { diff --git a/src/rlp/rlptraits.rs b/src/rlp/rlptraits.rs index a38cf76e2..4ef8d2a53 100644 --- a/src/rlp/rlptraits.rs +++ b/src/rlp/rlptraits.rs @@ -32,11 +32,11 @@ pub trait View<'a, 'view>: Sized { /// fn main () { /// let data = vec![0xc8, 0x83, b'c', b'a', b't', 0x83, b'd', b'o', b'g']; /// let rlp = Rlp::new(&data); - /// let dog = rlp.at(1).raw(); + /// let dog = rlp.at(1).as_raw(); /// assert_eq!(dog, &[0x83, b'd', b'o', b'g']); /// } /// ``` - fn raw(&'view self) -> &'a [u8]; + fn as_raw(&'view self) -> &'a [u8]; /// Get the prototype of the RLP. fn prototype(&self) -> Self::Prototype; diff --git a/src/rlp/tests.rs b/src/rlp/tests.rs index 49698dafb..7c2099124 100644 --- a/src/rlp/tests.rs +++ b/src/rlp/tests.rs @@ -19,19 +19,19 @@ fn rlp_at() { let cat = rlp.at(0).unwrap(); assert!(cat.is_data()); - assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + assert_eq!(cat.as_raw(), &[0x83, b'c', b'a', b't']); //assert_eq!(String::decode_untrusted(&cat).unwrap(), "cat".to_string()); assert_eq!(cat.as_val::().unwrap(), "cat".to_string()); let dog = rlp.at(1).unwrap(); assert!(dog.is_data()); - assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + assert_eq!(dog.as_raw(), &[0x83, b'd', b'o', b'g']); //assert_eq!(String::decode_untrusted(&dog).unwrap(), "dog".to_string()); assert_eq!(dog.as_val::().unwrap(), "dog".to_string()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_data()); - assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + assert_eq!(cat_again.as_raw(), &[0x83, b'c', b'a', b't']); //assert_eq!(String::decode_untrusted(&cat_again).unwrap(), "cat".to_string()); assert_eq!(cat_again.as_val::().unwrap(), "cat".to_string()); } @@ -61,18 +61,18 @@ fn rlp_iter() { let cat = iter.next().unwrap(); assert!(cat.is_data()); - assert_eq!(cat.raw(), &[0x83, b'c', b'a', b't']); + assert_eq!(cat.as_raw(), &[0x83, b'c', b'a', b't']); let dog = iter.next().unwrap(); assert!(dog.is_data()); - assert_eq!(dog.raw(), &[0x83, b'd', b'o', b'g']); + assert_eq!(dog.as_raw(), &[0x83, b'd', b'o', b'g']); let none = iter.next(); assert!(none.is_none()); let cat_again = rlp.at(0).unwrap(); assert!(cat_again.is_data()); - assert_eq!(cat_again.raw(), &[0x83, b'c', b'a', b't']); + assert_eq!(cat_again.as_raw(), &[0x83, b'c', b'a', b't']); } } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 2afbbfa23..5a12cbc5e 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -79,7 +79,7 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { } } - fn raw(&'view self) -> &'a [u8] { + fn as_raw(&'view self) -> &'a [u8] { self.bytes } @@ -294,7 +294,7 @@ impl<'a> Decoder for BasicDecoder<'a> { fn read_value(&self, f: F) -> Result where F: FnOnce(&[u8]) -> Result { - let bytes = self.rlp.raw(); + let bytes = self.rlp.as_raw(); match bytes.first().map(|&x| x) { // rlp is too short @@ -316,7 +316,7 @@ impl<'a> Decoder for BasicDecoder<'a> { } fn as_raw(&self) -> &[u8] { - self.rlp.raw() + self.rlp.as_raw() } fn as_list(&self) -> Result, DecoderError> { diff --git a/src/trie/node.rs b/src/trie/node.rs index 03f35a86d..b5745b66f 100644 --- a/src/trie/node.rs +++ b/src/trie/node.rs @@ -25,13 +25,13 @@ impl<'a> Node<'a> { // fed back into this function or inline RLP which can be fed back into this function). Prototype::List(2) => match NibbleSlice::from_encoded(r.at(0).data()) { (slice, true) => Node::Leaf(slice, r.at(1).data()), - (slice, false) => Node::Extension(slice, r.at(1).raw()), + (slice, false) => Node::Extension(slice, r.at(1).as_raw()), }, // branch - first 16 are nodes, 17th is a value (or empty). Prototype::List(17) => { let mut nodes: [&'a [u8]; 16] = unsafe { ::std::mem::uninitialized() }; for i in 0..16 { - nodes[i] = r.at(i).raw(); + nodes[i] = r.at(i).as_raw(); } Node::Branch(nodes, if r.at(16).is_empty() { None } else { Some(r.at(16).data()) }) }, diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index e04b7c758..783111f3a 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -309,22 +309,22 @@ impl<'db> TrieDBMut<'db> { /// removal instructions from the backing database. fn take_node<'a, 'rlp_view>(&'a self, rlp: &'rlp_view Rlp<'a>, journal: &mut Journal) -> &'a [u8] where 'a: 'rlp_view { if rlp.is_list() { - trace!("take_node {:?} (inline)", rlp.raw().pretty()); - rlp.raw() + trace!("take_node {:?} (inline)", rlp.as_raw().pretty()); + rlp.as_raw() } else if rlp.is_data() && rlp.size() == 32 { let h = rlp.as_val(); let r = self.db.lookup(&h).unwrap_or_else(||{ - println!("Node not found! rlp={:?}, node_hash={:?}", rlp.raw().pretty(), h); + println!("Node not found! rlp={:?}, node_hash={:?}", rlp.as_raw().pretty(), h); println!("Journal: {:?}", journal); panic!(); }); - trace!("take_node {:?} (indirect for {:?})", rlp.raw().pretty(), r); + trace!("take_node {:?} (indirect for {:?})", rlp.as_raw().pretty(), r); journal.delete_node_sha3(h); r } else { - trace!("take_node {:?} (???)", rlp.raw().pretty()); + trace!("take_node {:?} (???)", rlp.as_raw().pretty()); panic!("Empty or invalid node given?"); } } @@ -350,7 +350,7 @@ impl<'db> TrieDBMut<'db> { for i in 0..17 { match index == i { // not us - leave alone. - false => { s.append_raw(old_rlp.at(i).raw(), 1); }, + false => { s.append_raw(old_rlp.at(i).as_raw(), 1); }, // branch-leaf entry - just replace. true if i == 16 => { s.append(&value); }, // original had empty slot - place a leaf there. @@ -384,13 +384,13 @@ impl<'db> TrieDBMut<'db> { // not us - empty. _ if index != i => { s.append_empty_data(); }, // branch-value: just replace. - true if i == 16 => { s.append_raw(old_rlp.at(1).raw(), 1); }, + true if i == 16 => { s.append_raw(old_rlp.at(1).as_raw(), 1); }, // direct extension: just replace. - false if existing_key.len() == 1 => { s.append_raw(old_rlp.at(1).raw(), 1); }, + false if existing_key.len() == 1 => { s.append_raw(old_rlp.at(1).as_raw(), 1); }, // original has empty slot. true => journal.new_node(Self::compose_leaf(&existing_key.mid(1), old_rlp.at(1).data()), &mut s), // additional work required after branching. - false => journal.new_node(Self::compose_extension(&existing_key.mid(1), old_rlp.at(1).raw()), &mut s), + false => journal.new_node(Self::compose_extension(&existing_key.mid(1), old_rlp.at(1).as_raw()), &mut s), } }; self.augmented(&s.out(), partial, value, journal) @@ -422,7 +422,7 @@ impl<'db> TrieDBMut<'db> { trace!("partially-shared-prefix (exist={:?}; new={:?}; cp={:?}): AUGMENT-AT-END", existing_key.len(), partial.len(), cp); // low (farther from root) - let low = Self::compose_raw(&existing_key.mid(cp), old_rlp.at(1).raw(), is_leaf); + let low = Self::compose_raw(&existing_key.mid(cp), old_rlp.at(1).as_raw(), is_leaf); let augmented_low = self.augmented(&low, &partial.mid(cp), value, journal); // high (closer to root) diff --git a/src/uint.rs b/src/uint.rs index 6b2af4b3b..7fc11e2df 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -63,6 +63,28 @@ macro_rules! construct_uint { let &$name(ref arr) = self; arr[0] } + + /// Conversion to u32 with overflow checking + #[inline] + pub fn as_u32(&self) -> u32 { + let &$name(ref arr) = self; + if (arr[0] & (0xffffffffu64 << 32)) != 0 { + panic!("Intger overflow when casting U256") + } + self.as_u64() as u32 + } + + /// Conversion to u64 with overflow checking + #[inline] + pub fn as_u64(&self) -> u64 { + let &$name(ref arr) = self; + for i in 1..$n_words { + if arr[i] != 0 { + panic!("Intger overflow when casting U256") + } + } + arr[0] + } /// Return the least number of bits needed to represent the number #[inline] pub fn bits(&self) -> usize { @@ -442,13 +464,13 @@ impl From for U256 { impl From for u64 { fn from(value: U256) -> u64 { - value.low_u64() + value.as_u64() } } impl From for u32 { fn from(value: U256) -> u32 { - value.low_u32() + value.as_u32() } } From 647ea286ba33ec5517b2752bfd5e6c5c2e1ee6cf Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 8 Jan 2016 15:57:50 +0100 Subject: [PATCH 302/379] Renamed RlpStream::raw to as_raw --- src/rlp/rlpstream.rs | 2 +- src/rlp/rlptraits.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 0450c90b6..97ae1b484 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -101,7 +101,7 @@ impl Stream for RlpStream { self.unfinished_lists.len() == 0 } - fn raw(&self) -> &[u8] { + fn as_raw(&self) -> &[u8] { &self.encoder.bytes } diff --git a/src/rlp/rlptraits.rs b/src/rlp/rlptraits.rs index 4ef8d2a53..407d62daf 100644 --- a/src/rlp/rlptraits.rs +++ b/src/rlp/rlptraits.rs @@ -284,7 +284,7 @@ pub trait Stream: Sized { /// } fn is_finished(&self) -> bool; - fn raw(&self) -> &[u8]; + fn as_raw(&self) -> &[u8]; /// Streams out encoded bytes. /// From d7fe92242575d74b71aae05197f709b51e9116ff Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 8 Jan 2016 22:30:45 +0100 Subject: [PATCH 303/379] Fix API, --- src/trie/sectriedbmut.rs | 4 ++-- src/trie/triedbmut.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/trie/sectriedbmut.rs b/src/trie/sectriedbmut.rs index b0140582d..99c2e2bb1 100644 --- a/src/trie/sectriedbmut.rs +++ b/src/trie/sectriedbmut.rs @@ -22,8 +22,8 @@ impl<'db> SecTrieDBMut<'db> { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist - pub fn new_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { - SecTrieDBMut { raw: TrieDBMut::new_existing(db, root) } + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { + SecTrieDBMut { raw: TrieDBMut::from_existing(db, root) } } } diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index a51e56a83..b866bc988 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -70,7 +70,7 @@ impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist - pub fn new_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { + pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { assert!(db.exists(root)); TrieDBMut { db: db, @@ -1093,7 +1093,7 @@ mod tests { } { - let _ = TrieDBMut::new_existing(&mut db, &mut root); + let _ = TrieDBMut::from_existing(&mut db, &mut root); } } } From 49d97d7cb392d50971df0471be349d616196d4f2 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 10:20:07 +0100 Subject: [PATCH 304/379] is_zero for hash types --- src/hash.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/hash.rs b/src/hash.rs index a59813612..f9c43024c 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -30,6 +30,7 @@ pub trait FixedHash: Sized + BytesConvertable { fn bloom_part(&self, m: usize) -> T where T: FixedHash; fn contains_bloom(&self, b: &T) -> bool where T: FixedHash; fn contains<'a>(&'a self, b: &'a Self) -> bool; + fn is_zero(&self) -> bool; } macro_rules! impl_hash { @@ -165,6 +166,10 @@ macro_rules! impl_hash { fn contains<'a>(&'a self, b: &'a Self) -> bool { &(b & self) == b } + + fn is_zero(&self) -> bool { + self.eq(&Self::new()) + } } impl FromStr for $from { From 02c60acc14711fd66cf1bc975df34da28ce77bf3 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 10:22:03 +0100 Subject: [PATCH 305/379] Fixed build after merge --- src/sha3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sha3.rs b/src/sha3.rs index 7caaf1814..5b0a0c6a4 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -3,7 +3,7 @@ use std::mem::uninitialized; use tiny_keccak::Keccak; use bytes::{BytesConvertable,Populatable}; -use hash::H256; +use hash::{H256, FixedHash}; /// Types implementing this trait are sha3able. /// From 24ba723818ae0c855dfab25b0e39c3de4fb88d2e Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 9 Jan 2016 10:27:41 +0100 Subject: [PATCH 306/379] Do not expose sha3 from crypto --- src/crypto.rs | 2 -- src/network/discovery.rs | 1 + src/network/handshake.rs | 1 + src/network/host.rs | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/crypto.rs b/src/crypto.rs index a426cf9ba..a80045e55 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -2,8 +2,6 @@ use hash::*; use secp256k1::{key, Secp256k1}; use rand::os::OsRng; -pub use ::sha3::Hashable; - pub type Secret = H256; pub type Public = H512; pub type Signature = H520; diff --git a/src/network/discovery.rs b/src/network/discovery.rs index bae52eb10..f2d139f45 100644 --- a/src/network/discovery.rs +++ b/src/network/discovery.rs @@ -8,6 +8,7 @@ use std::ops::{DerefMut}; use mio::*; use mio::udp::*; use hash::*; +use sha3::Hashable; use crypto::*; use network::host::*; diff --git a/src/network/handshake.rs b/src/network/handshake.rs index 9f4ca4d97..4df8cbe8b 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -1,6 +1,7 @@ use mio::*; use mio::tcp::*; use hash::*; +use sha3::Hashable; use bytes::Bytes; use crypto::*; use crypto; diff --git a/src/network/host.rs b/src/network/host.rs index e8040a10e..9e2b3e101 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -8,6 +8,7 @@ use mio::tcp::*; use mio::udp::*; use hash::*; use crypto::*; +use sha3::Hashable; use rlp::*; use time::Tm; use network::handshake::Handshake; From 2b90f7d03a607ca1507951c6b9a931bd1f7e8384 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 9 Jan 2016 12:12:36 +0100 Subject: [PATCH 307/379] Introduce use-dependency amalgamation. We now have: - `use standard;` (bring in a set of standard and external dependencies) - `use common;` (bring in a low-level subset of this crate; basically anything that itself requires only `use standard;`) And, from an external create: - `use ethcore_util::*;` (bring in the entirety of this module and the standard dependencies) --- src/common.rs | 7 +++++++ src/lib.rs | 17 +++++++++++++++++ src/rlp/mod.rs | 4 ++-- src/rlp/{rlp.rs => rlpin.rs} | 0 src/standard.rs | 21 +++++++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/common.rs rename src/rlp/{rlp.rs => rlpin.rs} (100%) create mode 100644 src/standard.rs diff --git a/src/common.rs b/src/common.rs new file mode 100644 index 000000000..39294155c --- /dev/null +++ b/src/common.rs @@ -0,0 +1,7 @@ +pub use standard::*; +pub use error::*; +pub use hash::*; +pub use uint::*; +pub use bytes::*; +pub use vector::*; +pub use sha3::*; diff --git a/src/lib.rs b/src/lib.rs index 4859a8580..7cc4c35eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,8 @@ extern crate secp256k1; extern crate arrayvec; extern crate elastic_array; +pub mod standard; +pub mod common; pub mod error; pub mod hash; pub mod uint; @@ -68,4 +70,19 @@ pub mod heapsizeof; pub mod squeeze; pub mod semantic_version; +pub use common::*; +pub use rlp::*; +pub use hashdb::*; +pub use memorydb::*; +pub use overlaydb::*; +pub use math::*; +pub use chainfilter::*; +pub use crypto::*; +pub use triehash::*; +pub use trie::*; +pub use nibbleslice::*; +pub use heapsizeof::*; +pub use squeeze::*; +pub use semantic_version::*; + //pub mod network; diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 9c15259bd..bc2318fc4 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -32,7 +32,7 @@ pub mod rlptraits; pub mod rlperrors; -pub mod rlp; +pub mod rlpin; pub mod untrusted_rlp; pub mod rlpstream; @@ -42,7 +42,7 @@ mod tests; pub use self::rlperrors::DecoderError; pub use self::rlptraits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; -pub use self::rlp::{Rlp, RlpIterator}; +pub use self::rlpin::{Rlp, RlpIterator}; pub use self::rlpstream::{RlpStream}; use super::hash::H256; diff --git a/src/rlp/rlp.rs b/src/rlp/rlpin.rs similarity index 100% rename from src/rlp/rlp.rs rename to src/rlp/rlpin.rs diff --git a/src/standard.rs b/src/standard.rs new file mode 100644 index 000000000..b591220f3 --- /dev/null +++ b/src/standard.rs @@ -0,0 +1,21 @@ +pub use std::io; +pub use std::str; +pub use std::fmt; +pub use std::slice; + +pub use std::path::Path; +pub use std::str::{FromStr}; +pub use std::io::{Read,Write}; +pub use std::hash::{Hash, Hasher}; +pub use std::error::Error as StdError; + +pub use std::ops::*; +pub use std::cmp::*; +pub use std::cell::*; +pub use std::collections::*; + +pub use rustc_serialize::json::Json; +pub use rustc_serialize::base64::FromBase64; +pub use rustc_serialize::hex::FromHex; + +pub use heapsize::HeapSizeOf; From 568d28e94b5d0c64d8db30cb46fa8787dbeb619e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 9 Jan 2016 22:30:13 +0100 Subject: [PATCH 308/379] Bitwise Or Assign and some additional convenience in Hash. --- src/hash.rs | 22 +++++++++++++++++++++- src/lib.rs | 1 + src/triehash.rs | 7 ++----- src/uint.rs | 11 +++++++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 9eafa5dfb..78069a659 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use std::fmt; use std::ops; use std::hash::{Hash, Hasher}; -use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitAnd, BitXor}; +use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitOrAssign, BitAnd, BitXor}; use std::cmp::{PartialOrd, Ordering}; use rustc_serialize::hex::*; use error::EthcoreError; @@ -19,6 +19,8 @@ use uint::U256; /// Note: types implementing `FixedHash` must be also `BytesConvertable`. pub trait FixedHash: Sized + BytesConvertable + Populatable { fn new() -> Self; + /// Synonym for `new()`. Prefer to new as it's more readable. + fn zero() -> Self; fn random() -> Self; fn randomize(&mut self); fn size() -> usize; @@ -64,6 +66,10 @@ macro_rules! impl_hash { $from([0; $size]) } + fn zero() -> $from { + $from([0; $size]) + } + fn random() -> $from { let mut hash = $from::new(); hash.randomize(); @@ -299,6 +305,15 @@ macro_rules! impl_hash { } } + /// Moving BitOrAssign + impl<'a> BitOrAssign<&'a $from> for $from { + fn bitor_assign(&mut self, rhs: &'a Self) { + for i in 0..$size { + self.0[i] = self.0[i] | rhs.0[i]; + } + } + } + /// BitAnd on references impl <'a> BitAnd for &'a $from { type Output = $from; @@ -417,6 +432,11 @@ impl_hash!(H520, 65); impl_hash!(H1024, 128); impl_hash!(H2048, 256); +/// Constant address for point 0. Often used as a default. +pub static ZERO_ADDRESS: Address = Address([0x00; 20]); +/// Constant 256-bit datum for 0. Often used as a default. +pub static ZERO_H256: H256 = H256([0x00; 32]); + #[cfg(test)] mod tests { use hash::*; diff --git a/src/lib.rs b/src/lib.rs index 4827860c6..de2cdbdf7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(op_assign_traits)] //! Ethcore-util library //! //! ### Rust version: diff --git a/src/triehash.rs b/src/triehash.rs index de54bb3a7..66f9072b8 100644 --- a/src/triehash.rs +++ b/src/triehash.rs @@ -29,11 +29,8 @@ pub fn ordered_trie_root(input: Vec>) -> H256 { // first put elements into btree to sort them by nibbles // optimize it later .into_iter() - .fold(BTreeMap::new(), | mut acc, vec | { - let len = acc.len(); - acc.insert(rlp::encode(&len), vec); - acc - }) + .enumerate() + .fold(BTreeMap::new(), | mut acc, (i, vec) | { acc.insert(rlp::encode(&i), vec); acc }) // then move them to a vector .into_iter() .map(|(k, v)| (as_nibbles(&k), v) ) diff --git a/src/uint.rs b/src/uint.rs index 7fc11e2df..6fc8d19e1 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -424,6 +424,17 @@ macro_rules! construct_uint { } } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let &$name(ref data) = self; + try!(write!(f, "0x")); + for ch in data.iter().rev() { + try!(write!(f, "{:02x}", ch)); + } + Ok(()) + } + } + impl Hash for $name { fn hash(&self, state: &mut H) where H: Hasher { unsafe { state.write(::std::slice::from_raw_parts(self.0.as_ptr() as *mut u8, self.0.len() * 8)); } From ffc28bf49578e233b4ce259e406bf41b0a251ace Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 9 Jan 2016 23:47:46 +0100 Subject: [PATCH 309/379] Additional bloom-related functionality. RlpStandard for non-emit_*-based RlpStream extesions. --- src/chainfilter.rs | 8 ++++---- src/hash.rs | 30 +++++++++++++++++------------- src/rlp/mod.rs | 2 +- src/rlp/rlpstream.rs | 18 +++++++++++++++++- 4 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/chainfilter.rs b/src/chainfilter.rs index 076263fbb..e1804c191 100644 --- a/src/chainfilter.rs +++ b/src/chainfilter.rs @@ -18,7 +18,7 @@ //! let filter = ChainFilter::new(&cache, index_size, bloom_levels); //! let block_number = 39; //! let mut bloom = H2048::new(); -//! bloom.shift_bloom(&address.sha3()); +//! bloom.shift_bloomed(&address.sha3()); //! filter.add_bloom(&bloom, block_number) //! }; //! @@ -296,14 +296,14 @@ impl<'a, D> ChainFilter<'a, D> where D: FilterDataSource /// Returns numbers of blocks that may contain Address. pub fn blocks_with_address(&self, address: &Address, from_block: usize, to_block: usize) -> Vec { let mut bloom = H2048::new(); - bloom.shift_bloom(&address.sha3()); + bloom.shift_bloomed(&address.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) } /// Returns numbers of blocks that may contain Topic. pub fn blocks_with_topic(&self, topic: &H256, from_block: usize, to_block: usize) -> Vec { let mut bloom = H2048::new(); - bloom.shift_bloom(&topic.sha3()); + bloom.shift_bloomed(&topic.sha3()); self.blocks_with_bloom(&bloom, from_block, to_block) } @@ -415,7 +415,7 @@ mod tests { let filter = ChainFilter::new(&cache, index_size, bloom_levels); let block_number = 23; let mut bloom = H2048::new(); - bloom.shift_bloom(&topic.sha3()); + bloom.shift_bloomed(&topic.sha3()); filter.add_bloom(&bloom, block_number) }; diff --git a/src/hash.rs b/src/hash.rs index 78069a659..e368e064a 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -27,9 +27,10 @@ pub trait FixedHash: Sized + BytesConvertable + Populatable { fn from_slice(src: &[u8]) -> Self; fn clone_from_slice(&mut self, src: &[u8]) -> usize; fn copy_to(&self, dest: &mut [u8]); - fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; + fn shift_bloomed<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash; + fn with_bloomed(mut self, b: &T) -> Self where T: FixedHash { self.shift_bloomed(b); self } fn bloom_part(&self, m: usize) -> T where T: FixedHash; - fn contains_bloom(&self, b: &T) -> bool where T: FixedHash; + fn contains_bloomed(&self, b: &T) -> bool where T: FixedHash; fn contains<'a>(&'a self, b: &'a Self) -> bool; fn is_zero(&self) -> bool; } @@ -109,11 +110,12 @@ macro_rules! impl_hash { } } - fn shift_bloom<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash { + fn shift_bloomed<'a, T>(&'a mut self, b: &T) -> &'a mut Self where T: FixedHash { let bp: Self = b.bloom_part($size); let new_self = &bp | self; // impl |= instead + // TODO: that's done now! unsafe { use std::{mem, ptr}; @@ -159,7 +161,7 @@ macro_rules! impl_hash { ret } - fn contains_bloom(&self, b: &T) -> bool where T: FixedHash { + fn contains_bloomed(&self, b: &T) -> bool where T: FixedHash { let bp: Self = b.bloom_part($size); self.contains(&bp) } @@ -367,6 +369,8 @@ macro_rules! impl_hash { pub fn hex(&self) -> String { format!("{}", self) } + + pub fn from_bloomed(b: &T) -> Self where T: FixedHash { b.bloom_part($size) } } } } @@ -468,7 +472,7 @@ mod tests { } #[test] - fn shift_bloom() { + fn shift_bloomed() { use sha3::Hashable; let bloom = H2048::from_str("00000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002020000000000000000000000000000000000000000000008000000001000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000").unwrap(); @@ -476,17 +480,17 @@ mod tests { let topic = H256::from_str("02c69be41d0b7e40352fc85be1cd65eb03d40ef8427a0ca4596b1ead9a00e9fc").unwrap(); let mut my_bloom = H2048::new(); - assert!(!my_bloom.contains_bloom(&address.sha3())); - assert!(!my_bloom.contains_bloom(&topic.sha3())); + assert!(!my_bloom.contains_bloomed(&address.sha3())); + assert!(!my_bloom.contains_bloomed(&topic.sha3())); - my_bloom.shift_bloom(&address.sha3()); - assert!(my_bloom.contains_bloom(&address.sha3())); - assert!(!my_bloom.contains_bloom(&topic.sha3())); + my_bloom.shift_bloomed(&address.sha3()); + assert!(my_bloom.contains_bloomed(&address.sha3())); + assert!(!my_bloom.contains_bloomed(&topic.sha3())); - my_bloom.shift_bloom(&topic.sha3()); + my_bloom.shift_bloomed(&topic.sha3()); assert_eq!(my_bloom, bloom); - assert!(my_bloom.contains_bloom(&address.sha3())); - assert!(my_bloom.contains_bloom(&topic.sha3())); + assert!(my_bloom.contains_bloomed(&address.sha3())); + assert!(my_bloom.contains_bloomed(&topic.sha3())); } #[test] diff --git a/src/rlp/mod.rs b/src/rlp/mod.rs index 39cc3a7e7..fc0a4d288 100644 --- a/src/rlp/mod.rs +++ b/src/rlp/mod.rs @@ -43,7 +43,7 @@ pub use self::rlperrors::DecoderError; pub use self::rlptraits::{Decoder, Decodable, View, Stream, Encodable, Encoder}; pub use self::untrusted_rlp::{UntrustedRlp, UntrustedRlpIterator, PayloadInfo, Prototype}; pub use self::rlpin::{Rlp, RlpIterator}; -pub use self::rlpstream::{RlpStream}; +pub use self::rlpstream::{RlpStream,RlpStandard}; use super::hash::H256; pub const NULL_RLP: [u8; 1] = [0x80; 1]; diff --git a/src/rlp/rlpstream.rs b/src/rlp/rlpstream.rs index 97ae1b484..b8954ae6f 100644 --- a/src/rlp/rlpstream.rs +++ b/src/rlp/rlpstream.rs @@ -1,6 +1,8 @@ use elastic_array::*; -use bytes::ToBytes; +use bytes::{Bytes, ToBytes}; use rlp::{Stream, Encoder, Encodable}; +use hash::H256; +use sha3::*; #[derive(Debug, Copy, Clone)] struct ListInfo { @@ -213,6 +215,20 @@ impl Encoder for BasicEncoder { } } +pub trait RlpStandard { + fn rlp_append(&self, s: &mut RlpStream); + + fn rlp_bytes(&self) -> Bytes { + let mut s = RlpStream::new(); + self.rlp_append(&mut s); + s.out() + } + + fn rlp_sha3(&self) -> H256 { self.rlp_bytes().sha3() } +} + +// @debris TODO: implement Encoder for RlpStandard. + impl Encodable for T where T: ToBytes { fn encode(&self, encoder: &mut E) where E: Encoder { encoder.emit_value(&self.to_bytes()) From 5862d58379ba5a00d93a8ac1be4b034f566e9336 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 10 Jan 2016 12:53:55 +0100 Subject: [PATCH 310/379] Reworked network errors (now `NetworkError`) to be a subset of `UtilError`s rather than the other way around. --- src/error.rs | 60 ++++++++++++++++++++++++++++++++------- src/hash.rs | 9 +++--- src/network/connection.rs | 25 ++++++++-------- src/network/handshake.rs | 23 ++++++++------- src/network/host.rs | 19 +++++++------ src/network/mod.rs | 44 ++++++---------------------- src/network/service.rs | 9 +++--- src/network/session.rs | 39 ++++++++++++------------- src/overlaydb.rs | 2 +- 9 files changed, 123 insertions(+), 107 deletions(-) diff --git a/src/error.rs b/src/error.rs index 24b0cc87e..fbe57a98b 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,8 @@ //! General error types for use in ethcore. -use rustc_serialize::hex::*; +use rustc_serialize::hex::FromHexError; +use network::NetworkError; +use rlp::DecoderError; #[derive(Debug)] pub enum BaseDataError { @@ -9,22 +11,58 @@ pub enum BaseDataError { #[derive(Debug)] /// General error type which should be capable of representing all errors in ethcore. -pub enum EthcoreError { +pub enum UtilError { + Crypto(::crypto::CryptoError), + Io(::std::io::Error), + AddressParse(::std::net::AddrParseError), + AddressResolve(Option<::std::io::Error>), FromHex(FromHexError), BaseData(BaseDataError), + Network(NetworkError), + Decoder(DecoderError), BadSize, UnknownName, } -impl From for EthcoreError { - fn from(err: FromHexError) -> EthcoreError { - EthcoreError::FromHex(err) +impl From for UtilError { + fn from(err: FromHexError) -> UtilError { + UtilError::FromHex(err) } } -impl From for EthcoreError { - fn from(err: BaseDataError) -> EthcoreError { - EthcoreError::BaseData(err) +impl From for UtilError { + fn from(err: BaseDataError) -> UtilError { + UtilError::BaseData(err) + } +} + +impl From for UtilError { + fn from(err: NetworkError) -> UtilError { + UtilError::Network(err) + } +} + +impl From<::std::io::Error> for UtilError { + fn from(err: ::std::io::Error) -> UtilError { + UtilError::Io(err) + } +} + +impl From<::crypto::CryptoError> for UtilError { + fn from(err: ::crypto::CryptoError) -> UtilError { + UtilError::Crypto(err) + } +} + +impl From<::std::net::AddrParseError> for UtilError { + fn from(err: ::std::net::AddrParseError) -> UtilError { + UtilError::AddressParse(err) + } +} + +impl From<::rlp::DecoderError> for UtilError { + fn from(err: ::rlp::DecoderError) -> UtilError { + UtilError::Decoder(err) } } @@ -32,9 +70,9 @@ impl From for EthcoreError { /*#![feature(concat_idents)] macro_rules! assimilate { ($name:ident) => ( - impl From for EthcoreError { - fn from(err: concat_idents!($name, Error)) -> EthcoreError { - EthcoreError:: $name (err) + impl From for Error { + fn from(err: concat_idents!($name, Error)) -> Error { + Error:: $name (err) } } ) diff --git a/src/hash.rs b/src/hash.rs index e368e064a..087454ceb 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -7,7 +7,7 @@ use std::hash::{Hash, Hasher}; use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitOrAssign, BitAnd, BitXor}; use std::cmp::{PartialOrd, Ordering}; use rustc_serialize::hex::*; -use error::EthcoreError; +use error::UtilError; use rand::Rng; use rand::os::OsRng; use bytes::{BytesConvertable,Populatable}; @@ -176,11 +176,10 @@ macro_rules! impl_hash { } impl FromStr for $from { - type Err = EthcoreError; - - fn from_str(s: &str) -> Result<$from, EthcoreError> { + type Err = UtilError; + fn from_str(s: &str) -> Result<$from, UtilError> { let a = try!(s.from_hex()); - if a.len() != $size { return Err(EthcoreError::BadSize); } + if a.len() != $size { return Err(UtilError::BadSize); } let mut ret = $from([0;$size]); for i in 0..$size { ret.0[i] = a[i]; diff --git a/src/network/connection.rs b/src/network/connection.rs index 2a8025b83..2db53c11e 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -7,7 +7,8 @@ use bytes::*; use rlp::*; use std::io::{self, Cursor, Read}; use network::host::{Host}; -use network::Error; +use error::*; +use network::NetworkError; use network::handshake::Handshake; use crypto; use rcrypto::blockmodes::*; @@ -158,7 +159,7 @@ pub struct EncryptedConnection { } impl EncryptedConnection { - pub fn new(handshake: Handshake) -> Result { + pub fn new(handshake: Handshake) -> Result { let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); let mut nonce_material = H512::new(); if handshake.originated { @@ -207,7 +208,7 @@ impl EncryptedConnection { }) } - pub fn send_packet(&mut self, payload: &[u8]) -> Result<(), Error> { + pub fn send_packet(&mut self, payload: &[u8]) -> Result<(), UtilError> { let mut header = RlpStream::new(); let len = payload.len() as usize; header.append_raw(&[(len >> 16) as u8, (len >> 8) as u8, len as u8], 1); @@ -233,16 +234,16 @@ impl EncryptedConnection { Ok(()) } - fn read_header(&mut self, header: &[u8]) -> Result<(), Error> { + fn read_header(&mut self, header: &[u8]) -> Result<(), UtilError> { if header.len() != ENCRYPTED_HEADER_LEN { - return Err(Error::Auth); + return Err(From::from(NetworkError::Auth)); } EncryptedConnection::update_mac(&mut self.ingress_mac, &mut self.mac_encoder, &header[0..16]); let mac = &header[16..]; let mut expected = H256::new(); self.ingress_mac.clone().finalize(&mut expected); if mac != &expected[0..16] { - return Err(Error::Auth); + return Err(From::from(NetworkError::Auth)); } let mut hdec = H128::new(); @@ -262,11 +263,11 @@ impl EncryptedConnection { Ok(()) } - fn read_payload(&mut self, payload: &[u8]) -> Result { + fn read_payload(&mut self, payload: &[u8]) -> Result { let padding = (16 - (self.payload_len % 16)) % 16; let full_length = (self.payload_len + padding + 16) as usize; if payload.len() != full_length { - return Err(Error::Auth); + return Err(From::from(NetworkError::Auth)); } self.ingress_mac.update(&payload[0..payload.len() - 16]); EncryptedConnection::update_mac(&mut self.ingress_mac, &mut self.mac_encoder, &[0u8; 0]); @@ -274,7 +275,7 @@ impl EncryptedConnection { let mut expected = H128::new(); self.ingress_mac.clone().finalize(&mut expected); if mac != &expected[..] { - return Err(Error::Auth); + return Err(From::from(NetworkError::Auth)); } let mut packet = vec![0u8; self.payload_len as usize]; @@ -297,7 +298,7 @@ impl EncryptedConnection { mac.update(&enc); } - pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, Error> { + pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.reregister(event_loop)); match self.read_state { @@ -323,14 +324,14 @@ impl EncryptedConnection { } } - pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); try!(self.connection.reregister(event_loop)); Ok(()) } - pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.connection.expect(ENCRYPTED_HEADER_LEN); self.idle_timeout.map(|t| event_loop.clear_timeout(t)); self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); diff --git a/src/network/handshake.rs b/src/network/handshake.rs index 4df8cbe8b..d9e280bfe 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -7,7 +7,8 @@ use crypto::*; use crypto; use network::connection::{Connection}; use network::host::{NodeId, Host, HostInfo}; -use network::Error; +use error::*; +use network::NetworkError; #[derive(PartialEq, Eq, Debug)] enum HandshakeState { @@ -35,7 +36,7 @@ const AUTH_PACKET_SIZE: usize = 307; const ACK_PACKET_SIZE: usize = 210; impl Handshake { - pub fn new(token: Token, id: &NodeId, socket: TcpStream, nonce: &H256) -> Result { + pub fn new(token: Token, id: &NodeId, socket: TcpStream, nonce: &H256) -> Result { Ok(Handshake { id: id.clone(), connection: Connection::new(token, socket), @@ -51,7 +52,7 @@ impl Handshake { }) } - pub fn start(&mut self, host: &HostInfo, originated: bool) -> Result<(), Error> { + pub fn start(&mut self, host: &HostInfo, originated: bool) -> Result<(), UtilError> { self.originated = originated; if originated { try!(self.write_auth(host)); @@ -67,7 +68,7 @@ impl Handshake { self.state == HandshakeState::StartSession } - pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), Error> { + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); match self.state { HandshakeState::ReadingAuth => { @@ -96,7 +97,7 @@ impl Handshake { Ok(()) } - pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); if self.state != HandshakeState::StartSession { @@ -105,14 +106,14 @@ impl Handshake { Ok(()) } - pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); try!(self.connection.register(event_loop)); Ok(()) } - fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), Error> { + fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); assert!(data.len() == AUTH_PACKET_SIZE); self.auth_cipher = data.to_vec(); @@ -128,12 +129,12 @@ impl Handshake { let spub = try!(ec::recover(&signature, &(&shared ^ &self.remote_nonce))); if &spub.sha3()[..] != hepubk { trace!(target:"net", "Handshake hash mismath with {:?}", self.connection.socket.peer_addr()); - return Err(Error::Auth); + return Err(From::from(NetworkError::Auth)); }; self.write_ack() } - fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), Error> { + fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); assert!(data.len() == ACK_PACKET_SIZE); self.ack_cipher = data.to_vec(); @@ -143,7 +144,7 @@ impl Handshake { Ok(()) } - fn write_auth(&mut self, host: &HostInfo) -> Result<(), Error> { + fn write_auth(&mut self, host: &HostInfo) -> Result<(), UtilError> { trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants let len = data.len(); @@ -169,7 +170,7 @@ impl Handshake { Ok(()) } - fn write_ack(&mut self) -> Result<(), Error> { + fn write_ack(&mut self) -> Result<(), UtilError> { trace!(target:"net", "Sending handshake ack to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants let len = data.len(); diff --git a/src/network/host.rs b/src/network/host.rs index 9e2b3e101..243813b63 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -13,7 +13,8 @@ use rlp::*; use time::Tm; use network::handshake::Handshake; use network::session::{Session, SessionData}; -use network::{Error, ProtocolHandler}; +use error::*; +use network::ProtocolHandler; const _DEFAULT_PORT: u16 = 30304; @@ -53,7 +54,7 @@ pub struct NodeEndpoint { } impl NodeEndpoint { - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { let address = s.to_socket_addrs().map(|mut i| i.next()); match address { Ok(Some(a)) => Ok(NodeEndpoint { @@ -61,8 +62,8 @@ impl NodeEndpoint { address_str: s.to_string(), udp_port: a.port() }), - Ok(_) => Err(Error::AddressResolve(None)), - Err(e) => Err(Error::AddressResolve(Some(e))) + Ok(_) => Err(UtilError::AddressResolve(None)), + Err(e) => Err(UtilError::AddressResolve(Some(e))) } } } @@ -81,7 +82,7 @@ struct Node { } impl FromStr for Node { - type Err = Error; + type Err = UtilError; fn from_str(s: &str) -> Result { let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) @@ -189,7 +190,7 @@ impl<'s> HostIo<'s> { } /// Send a packet over the network to another peer. - pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), Error> { + pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { match self.connections.get_mut(Token(peer)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { @@ -204,7 +205,7 @@ impl<'s> HostIo<'s> { } /// Respond to a current network message. Panics if no there is no packet in the context. - pub fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), Error> { + pub fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { match self.session { Some(session) => self.send(session.as_usize(), packet_id, data), None => { @@ -214,7 +215,7 @@ impl<'s> HostIo<'s> { } /// Register a new IO timer. Returns a new timer toke. 'ProtocolHandler::timeout' will be called with the token. - pub fn register_timer(&mut self, ms: u64) -> Result{ + pub fn register_timer(&mut self, ms: u64) -> Result{ match self.timers.insert(UserTimer { delay: ms, protocol: self.protocol, @@ -292,7 +293,7 @@ pub struct Host { } impl Host { - pub fn start(event_loop: &mut EventLoop) -> Result<(), Error> { + pub fn start(event_loop: &mut EventLoop) -> Result<(), UtilError> { let config = NetworkConfiguration::new(); /* match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { diff --git a/src/network/mod.rs b/src/network/mod.rs index df0da2c13..cdce08d00 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -69,49 +69,23 @@ pub enum DisconnectReason } #[derive(Debug)] -pub enum Error { - Crypto(::crypto::CryptoError), - Io(::std::io::Error), +pub enum NetworkError { Auth, BadProtocol, - AddressParse(::std::net::AddrParseError), - AddressResolve(Option<::std::io::Error>), - NodeIdParse(::error::EthcoreError), PeerNotFound, - Disconnect(DisconnectReason) + Disconnect(DisconnectReason), + Mio(::std::io::Error), } -impl From<::std::io::Error> for Error { - fn from(err: ::std::io::Error) -> Error { - Error::Io(err) +impl From<::rlp::DecoderError> for NetworkError { + fn from(_err: ::rlp::DecoderError) -> NetworkError { + NetworkError::Auth } } -impl From<::crypto::CryptoError> for Error { - fn from(err: ::crypto::CryptoError) -> Error { - Error::Crypto(err) - } -} - -impl From<::std::net::AddrParseError> for Error { - fn from(err: ::std::net::AddrParseError) -> Error { - Error::AddressParse(err) - } -} -impl From<::error::EthcoreError> for Error { - fn from(err: ::error::EthcoreError) -> Error { - Error::NodeIdParse(err) - } -} -impl From<::rlp::DecoderError> for Error { - fn from(_err: ::rlp::DecoderError) -> Error { - Error::Auth - } -} - -impl From<::mio::NotifyError> for Error { - fn from(_err: ::mio::NotifyError) -> Error { - Error::Io(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) +impl From<::mio::NotifyError> for NetworkError { + fn from(_err: ::mio::NotifyError) -> NetworkError { + NetworkError::Mio(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) } } diff --git a/src/network/service.rs b/src/network/service.rs index 7598ffdd6..23cbb57d9 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -1,6 +1,7 @@ use std::thread::{self, JoinHandle}; use mio::*; -use network::{Error, ProtocolHandler}; +use error::*; +use network::{NetworkError, ProtocolHandler}; use network::host::{Host, HostMessage, PeerId, PacketId, ProtocolId}; /// IO Service with networking @@ -11,7 +12,7 @@ pub struct NetworkService { impl NetworkService { /// Starts IO event loop - pub fn start() -> Result { + pub fn start() -> Result { let mut event_loop = EventLoop::new().unwrap(); let channel = event_loop.channel(); let thread = thread::spawn(move || { @@ -24,7 +25,7 @@ impl NetworkService { } /// Send a message over the network. Normaly `HostIo::send` should be used. This can be used from non-io threads. - pub fn send(&mut self, peer: &PeerId, packet_id: PacketId, protocol: ProtocolId, data: &[u8]) -> Result<(), Error> { + pub fn send(&mut self, peer: &PeerId, packet_id: PacketId, protocol: ProtocolId, data: &[u8]) -> Result<(), NetworkError> { try!(self.host_channel.send(HostMessage::Send { peer: *peer, packet_id: packet_id, @@ -35,7 +36,7 @@ impl NetworkService { } /// Regiter a new protocol handler with the event loop. - pub fn register_protocol(&mut self, handler: Box, protocol: ProtocolId, versions: &[u8]) -> Result<(), Error> { + pub fn register_protocol(&mut self, handler: Box, protocol: ProtocolId, versions: &[u8]) -> Result<(), NetworkError> { try!(self.host_channel.send(HostMessage::AddHandler { handler: handler, protocol: protocol, diff --git a/src/network/session.rs b/src/network/session.rs index 720902150..d37ec086f 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -3,7 +3,8 @@ use hash::*; use rlp::*; use network::connection::{EncryptedConnection, Packet}; use network::handshake::Handshake; -use network::{Error, DisconnectReason}; +use error::*; +use network::{NetworkError, DisconnectReason}; use network::host::*; pub struct Session { @@ -64,7 +65,7 @@ const PACKET_USER: u8 = 0x10; const PACKET_LAST: u8 = 0x7f; impl Session { - pub fn new(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { + pub fn new(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); let mut session = Session { @@ -83,14 +84,14 @@ impl Session { Ok(session) } - pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { match try!(self.connection.readable(event_loop)) { - Some(data) => self.read_packet(data, host), + Some(data) => Ok(try!(self.read_packet(data, host))), None => Ok(SessionData::None) } } - pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), Error> { + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { self.connection.writable(event_loop) } @@ -98,7 +99,7 @@ impl Session { self.info.capabilities.iter().any(|c| c.protocol == protocol) } - pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), Error> { + pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), UtilError> { let mut i = 0usize; while protocol != self.info.capabilities[i].protocol { i += 1; @@ -114,13 +115,13 @@ impl Session { self.connection.send_packet(&rlp.out()) } - fn read_packet(&mut self, packet: Packet, host: &HostInfo) -> Result { + fn read_packet(&mut self, packet: Packet, host: &HostInfo) -> Result { if packet.data.len() < 2 { - return Err(Error::BadProtocol); + return Err(From::from(NetworkError::BadProtocol)); } let packet_id = packet.data[0]; if packet_id != PACKET_HELLO && packet_id != PACKET_DISCONNECT && !self.had_hello { - return Err(Error::BadProtocol); + return Err(From::from(NetworkError::BadProtocol)); } match packet_id { PACKET_HELLO => { @@ -128,7 +129,7 @@ impl Session { try!(self.read_hello(&rlp, host)); Ok(SessionData::Ready) }, - PACKET_DISCONNECT => Err(Error::Disconnect(DisconnectReason::DisconnectRequested)), + PACKET_DISCONNECT => Err(From::from(NetworkError::Disconnect(DisconnectReason::DisconnectRequested))), PACKET_PING => { try!(self.write_pong()); Ok(SessionData::None) @@ -157,7 +158,7 @@ impl Session { } } - fn write_hello(&mut self, host: &HostInfo) -> Result<(), Error> { + fn write_hello(&mut self, host: &HostInfo) -> Result<(), UtilError> { let mut rlp = RlpStream::new(); rlp.append(&(PACKET_HELLO as u32)); rlp.append_list(5) @@ -169,7 +170,7 @@ impl Session { self.connection.send_packet(&rlp.out()) } - fn read_hello(&mut self, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), Error> { + fn read_hello(&mut self, rlp: &UntrustedRlp, host: &HostInfo) -> Result<(), UtilError> { let protocol = try!(rlp.val_at::(0)); let client_version = try!(rlp.val_at::(1)); let peer_caps = try!(rlp.val_at::>(2)); @@ -210,37 +211,37 @@ impl Session { trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); self.info.capabilities = caps; if protocol != host.protocol_version { - return Err(self.disconnect(DisconnectReason::UselessPeer)); + return Err(From::from(self.disconnect(DisconnectReason::UselessPeer))); } self.had_hello = true; Ok(()) } - fn write_ping(&mut self) -> Result<(), Error> { + fn write_ping(&mut self) -> Result<(), UtilError> { self.send(try!(Session::prepare(PACKET_PING, 0))) } - fn write_pong(&mut self) -> Result<(), Error> { + fn write_pong(&mut self) -> Result<(), UtilError> { self.send(try!(Session::prepare(PACKET_PONG, 0))) } - fn disconnect(&mut self, reason: DisconnectReason) -> Error { + fn disconnect(&mut self, reason: DisconnectReason) -> NetworkError { let mut rlp = RlpStream::new(); rlp.append(&(PACKET_DISCONNECT as u32)); rlp.append_list(1); rlp.append(&(reason.clone() as u32)); self.connection.send_packet(&rlp.out()).ok(); - Error::Disconnect(reason) + NetworkError::Disconnect(reason) } - fn prepare(packet_id: u8, items: usize) -> Result { + fn prepare(packet_id: u8, items: usize) -> Result { let mut rlp = RlpStream::new_list(1); rlp.append(&(packet_id as u32)); rlp.append_list(items); Ok(rlp) } - fn send(&mut self, rlp: RlpStream) -> Result<(), Error> { + fn send(&mut self, rlp: RlpStream) -> Result<(), UtilError> { self.connection.send_packet(&rlp.out()) } } diff --git a/src/overlaydb.rs b/src/overlaydb.rs index b5bbf4299..5539a8893 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -66,7 +66,7 @@ impl OverlayDB { /// assert!(m.exists(&key)); // key now still exists. /// } /// ``` - pub fn commit(&mut self) -> Result { + pub fn commit(&mut self) -> Result { let mut ret = 0u32; for i in self.overlay.drain().into_iter() { let (key, (value, rc)) = i; From b0cef968e7c900a04534eecd79e526fb4410f43c Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2016 14:02:01 +0100 Subject: [PATCH 311/379] Networking fixes --- src/network/connection.rs | 23 +++++++++++------- src/network/host.rs | 51 +++++++++++++++++++++++++++++++++------ src/network/session.rs | 20 ++++++++++----- src/rlp/untrusted_rlp.rs | 2 +- 4 files changed, 73 insertions(+), 23 deletions(-) diff --git a/src/network/connection.rs b/src/network/connection.rs index 2a8025b83..131aae318 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -154,7 +154,7 @@ pub struct EncryptedConnection { read_state: EncryptedConnectionState, idle_timeout: Option, protocol_id: u16, - payload_len: u32, + payload_len: usize, } impl EncryptedConnection { @@ -223,7 +223,7 @@ impl EncryptedConnection { self.egress_mac.clone().finalize(&mut packet[16..32]); self.encoder.encrypt(&mut RefReadBuffer::new(&payload), &mut RefWriteBuffer::new(&mut packet[32..(32 + len)]), padding == 0).expect("Invalid length or padding"); if padding != 0 { - let pad = [08; 16]; + let pad = [0u8; 16]; self.encoder.encrypt(&mut RefReadBuffer::new(&pad[0..padding]), &mut RefWriteBuffer::new(&mut packet[(32 + len)..(32 + len + padding)]), true).expect("Invalid length or padding"); } self.egress_mac.update(&packet[32..(32 + len + padding)]); @@ -252,7 +252,7 @@ impl EncryptedConnection { let header_rlp = UntrustedRlp::new(&hdec[3..6]); let protocol_id = try!(header_rlp.val_at::(0)); - self.payload_len = length; + self.payload_len = length as usize; self.protocol_id = protocol_id; self.read_state = EncryptedConnectionState::Payload; @@ -264,7 +264,7 @@ impl EncryptedConnection { fn read_payload(&mut self, payload: &[u8]) -> Result { let padding = (16 - (self.payload_len % 16)) % 16; - let full_length = (self.payload_len + padding + 16) as usize; + let full_length = self.payload_len + padding + 16; if payload.len() != full_length { return Err(Error::Auth); } @@ -277,9 +277,10 @@ impl EncryptedConnection { return Err(Error::Auth); } - let mut packet = vec![0u8; self.payload_len as usize]; - self.decoder.decrypt(&mut RefReadBuffer::new(&payload[0..(full_length - 16)]), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding"); - packet.resize(self.payload_len as usize, 0u8); + let mut packet = vec![0u8; self.payload_len]; + self.decoder.decrypt(&mut RefReadBuffer::new(&payload[0..self.payload_len]), &mut RefWriteBuffer::new(&mut packet), false).expect("Invalid length or padding"); + let mut pad_buf = [0u8; 16]; + self.decoder.decrypt(&mut RefReadBuffer::new(&payload[self.payload_len..(payload.len() - 16)]), &mut RefWriteBuffer::new(&mut pad_buf), false).expect("Invalid length or padding"); Ok(Packet { protocol: self.protocol_id, data: packet @@ -299,7 +300,6 @@ impl EncryptedConnection { pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); - try!(self.connection.reregister(event_loop)); match self.read_state { EncryptedConnectionState::Header => { match try!(self.connection.readable()) { @@ -326,7 +326,6 @@ impl EncryptedConnection { pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); - try!(self.connection.reregister(event_loop)); Ok(()) } @@ -337,6 +336,12 @@ impl EncryptedConnection { try!(self.connection.reregister(event_loop)); Ok(()) } + + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + try!(self.connection.reregister(event_loop)); + Ok(()) + } + } #[test] diff --git a/src/network/host.rs b/src/network/host.rs index 9e2b3e101..b9ca0152b 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -295,11 +295,10 @@ impl Host { pub fn start(event_loop: &mut EventLoop) -> Result<(), Error> { let config = NetworkConfiguration::new(); /* - match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { - Some(iface) => config.public_address = iface.addr.unwrap(), - None => warn!("No public network interface"), - } - */ + match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { + Some(iface) => config.public_address = iface.addr.unwrap(), + None => warn!("No public network interface"), + */ let addr = config.listen_address; // Setup the server socket @@ -487,8 +486,17 @@ impl Host { if create_session { self.start_session(token, event_loop); } + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.reregister(event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + }, + _ => (), + } } + fn connection_closed(&mut self, token: Token, event_loop: &mut EventLoop) { + self.kill_connection(token, event_loop); + } fn connection_readable(&mut self, token: Token, event_loop: &mut EventLoop) { let mut kill = false; @@ -549,6 +557,12 @@ impl Host { h.read(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize(), packet_id, &data[1..]); } + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Session(ref mut s)) => { + s.reregister(event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + }, + _ => (), + } } fn start_session(&mut self, token: Token, event_loop: &mut EventLoop) { @@ -570,7 +584,23 @@ impl Host { self.kill_connection(token, event_loop) } - fn kill_connection(&mut self, token: Token, _event_loop: &mut EventLoop) { + fn kill_connection(&mut self, token: Token, event_loop: &mut EventLoop) { + let mut to_disconnect: Vec = Vec::new(); + match self.connections.get_mut(token) { + Some(&mut ConnectionEntry::Handshake(_)) => (), // just abandon handshake + Some(&mut ConnectionEntry::Session(ref mut s)) if s.is_ready() => { + for (p, _) in self.handlers.iter_mut() { + if s.have_capability(p) { + to_disconnect.push(p); + } + } + }, + _ => (), + } + for p in to_disconnect { + let mut h = self.handlers.get_mut(p).unwrap(); + h.disconnected(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize()); + } self.connections.remove(token); } } @@ -580,7 +610,14 @@ impl Handler for Host { type Message = HostMessage; fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { - if events.is_readable() { + if events.is_hup() { + trace!(target: "net", "hup"); + match token.as_usize() { + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_closed(token, event_loop), + _ => warn!(target: "net", "Unexpected hup"), + }; + } + else if events.is_readable() { match token.as_usize() { TCP_ACCEPT => self.accept(event_loop), IDLE => self.maintain_network(event_loop), diff --git a/src/network/session.rs b/src/network/session.rs index 720902150..8e89ff2c4 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -83,6 +83,10 @@ impl Session { Ok(session) } + pub fn is_ready(&self) -> bool { + self.had_hello + } + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { match try!(self.connection.readable(event_loop)) { Some(data) => self.read_packet(data, host), @@ -98,6 +102,10 @@ impl Session { self.info.capabilities.iter().any(|c| c.protocol == protocol) } + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), Error> { + self.connection.reregister(event_loop) + } + pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), Error> { let mut i = 0usize; while protocol != self.info.capabilities[i].protocol { @@ -159,7 +167,7 @@ impl Session { fn write_hello(&mut self, host: &HostInfo) -> Result<(), Error> { let mut rlp = RlpStream::new(); - rlp.append(&(PACKET_HELLO as u32)); + rlp.append_raw(&[PACKET_HELLO as u8], 0); rlp.append_list(5) .append(&host.protocol_version) .append(&host.client_version) @@ -217,11 +225,11 @@ impl Session { } fn write_ping(&mut self) -> Result<(), Error> { - self.send(try!(Session::prepare(PACKET_PING, 0))) + self.send(try!(Session::prepare(PACKET_PING))) } fn write_pong(&mut self) -> Result<(), Error> { - self.send(try!(Session::prepare(PACKET_PONG, 0))) + self.send(try!(Session::prepare(PACKET_PONG))) } fn disconnect(&mut self, reason: DisconnectReason) -> Error { @@ -233,10 +241,10 @@ impl Session { Error::Disconnect(reason) } - fn prepare(packet_id: u8, items: usize) -> Result { - let mut rlp = RlpStream::new_list(1); + fn prepare(packet_id: u8) -> Result { + let mut rlp = RlpStream::new(); rlp.append(&(packet_id as u32)); - rlp.append_list(items); + rlp.append_list(0); Ok(rlp) } diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 5a12cbc5e..452a198bb 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -188,7 +188,7 @@ impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { } fn val_at(&self, index: usize) -> Result where T: Decodable { - self.at(index).unwrap().as_val() + try!(self.at(index)).as_val() } } From 6410759bc3bfbb9dc98c7d009bcdaf524347f894 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 10 Jan 2016 14:06:38 +0100 Subject: [PATCH 312/379] Remove unneeded error. --- src/error.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index fbe57a98b..17b65ceec 100644 --- a/src/error.rs +++ b/src/error.rs @@ -21,7 +21,6 @@ pub enum UtilError { Network(NetworkError), Decoder(DecoderError), BadSize, - UnknownName, } impl From for UtilError { From 02b530f4aae70c7b8b55062a4adf287031f3da1f Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2016 15:13:12 +0100 Subject: [PATCH 313/379] Style --- src/network/connection.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/network/connection.rs b/src/network/connection.rs index efa0059eb..3f5d420e9 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -342,7 +342,6 @@ impl EncryptedConnection { try!(self.connection.reregister(event_loop)); Ok(()) } - } #[test] From 6f3c3fa020816dd266e118ed4e532d81aca14027 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sun, 10 Jan 2016 22:42:27 +0100 Subject: [PATCH 314/379] Documentation --- src/network/connection.rs | 42 +++++++++++++++++++++++++++++++++++++++ src/network/handshake.rs | 26 ++++++++++++++++++++++++ src/network/host.rs | 39 ++++++++++++++++++++++++++++++++++-- src/network/session.rs | 28 ++++++++++++++++++++++++-- 4 files changed, 131 insertions(+), 4 deletions(-) diff --git a/src/network/connection.rs b/src/network/connection.rs index 3f5d420e9..2c671a08a 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -19,22 +19,33 @@ use tiny_keccak::Keccak; const ENCRYPTED_HEADER_LEN: usize = 32; +/// Low level tcp connection pub struct Connection { + /// Connection id (token) pub token: Token, + /// Network socket pub socket: TcpStream, + /// Receive buffer rec_buf: Bytes, + /// Expected size rec_size: usize, + /// Send out packets FIFO send_queue: VecDeque>, + /// Event flags this connection expects interest: EventSet, } +/// Connection write status. #[derive(PartialEq, Eq)] pub enum WriteStatus { + /// Some data is still pending for current packet Ongoing, + /// All data sent. Complete } impl Connection { + /// Create a new connection with given id and socket. pub fn new(token: Token, socket: TcpStream) -> Connection { Connection { token: token, @@ -46,6 +57,7 @@ impl Connection { } } + /// Put a connection into read mode. Receiving up `size` bytes of data. pub fn expect(&mut self, size: usize) { if self.rec_size != self.rec_buf.len() { warn!(target:"net", "Unexpected connection read start"); @@ -54,6 +66,7 @@ impl Connection { self.rec_size = size; } + /// Readable IO handler. Called when there is some data to be read. //TODO: return a slice pub fn readable(&mut self) -> io::Result> { if self.rec_size == 0 || self.rec_buf.len() >= self.rec_size { @@ -72,6 +85,7 @@ impl Connection { } } + /// Add a packet to send queue. pub fn send(&mut self, data: Bytes) { if data.len() != 0 { self.send_queue.push_back(Cursor::new(data)); @@ -81,6 +95,7 @@ impl Connection { } } + /// Writable IO handler. Called when the socket is ready to send. pub fn writable(&mut self) -> io::Result { if self.send_queue.is_empty() { return Ok(WriteStatus::Complete) @@ -117,6 +132,7 @@ impl Connection { }) } + /// Register this connection with the IO event loop. pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", self.token); self.interest.insert(EventSet::readable()); @@ -126,6 +142,7 @@ impl Connection { }) } + /// Update connection registration. Should be called at the end of the IO handler. pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection reregister; token={:?}", self.token); event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { @@ -135,30 +152,47 @@ impl Connection { } } +/// RLPx packet pub struct Packet { pub protocol: u16, pub data: Bytes, } +/// Encrypted connection receiving state. enum EncryptedConnectionState { + /// Reading a header. Header, + /// Reading the rest of the packet. Payload, } +/// Connection implementing RLPx framing +/// https://github.com/ethereum/devp2p/blob/master/rlpx.md#framing pub struct EncryptedConnection { + /// Underlying tcp connection connection: Connection, + /// Egress data encryptor encoder: CtrMode, + /// Ingress data decryptor decoder: CtrMode, + /// Ingress data decryptor mac_encoder: EcbEncryptor>, + /// MAC for egress data egress_mac: Keccak, + /// MAC for ingress data ingress_mac: Keccak, + /// Read state read_state: EncryptedConnectionState, + /// Disconnect timeout idle_timeout: Option, + /// Protocol id for the last received packet protocol_id: u16, + /// Payload expected to be received for the last header. payload_len: usize, } impl EncryptedConnection { + /// Create an encrypted connection out of the handshake. Consumes a handshake object. pub fn new(handshake: Handshake) -> Result { let shared = try!(crypto::ecdh::agree(handshake.ecdhe.secret(), &handshake.remote_public)); let mut nonce_material = H512::new(); @@ -208,6 +242,7 @@ impl EncryptedConnection { }) } + /// Send a packet pub fn send_packet(&mut self, payload: &[u8]) -> Result<(), UtilError> { let mut header = RlpStream::new(); let len = payload.len() as usize; @@ -234,6 +269,7 @@ impl EncryptedConnection { Ok(()) } + /// Decrypt and authenticate an incoming packet header. Prepare for receiving payload. fn read_header(&mut self, header: &[u8]) -> Result<(), UtilError> { if header.len() != ENCRYPTED_HEADER_LEN { return Err(From::from(NetworkError::Auth)); @@ -263,6 +299,7 @@ impl EncryptedConnection { Ok(()) } + /// Decrypt and authenticate packet payload. fn read_payload(&mut self, payload: &[u8]) -> Result { let padding = (16 - (self.payload_len % 16)) % 16; let full_length = self.payload_len + padding + 16; @@ -288,6 +325,7 @@ impl EncryptedConnection { }) } + /// Update MAC after reading or writing any data. fn update_mac(mac: &mut Keccak, mac_encoder: &mut EcbEncryptor>, seed: &[u8]) { let mut prev = H128::new(); mac.clone().finalize(&mut prev); @@ -299,6 +337,7 @@ impl EncryptedConnection { mac.update(&enc); } + /// Readable IO handler. Tracker receive status and returns decoded packet if avaialable. pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); match self.read_state { @@ -324,12 +363,14 @@ impl EncryptedConnection { } } + /// Writable IO handler. Processes send queeue. pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); Ok(()) } + /// Register this connection with the event handler. pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.connection.expect(ENCRYPTED_HEADER_LEN); self.idle_timeout.map(|t| event_loop.clear_timeout(t)); @@ -338,6 +379,7 @@ impl EncryptedConnection { Ok(()) } + /// Update connection registration. This should be called at the end of the event loop. pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { try!(self.connection.reregister(event_loop)); Ok(()) diff --git a/src/network/handshake.rs b/src/network/handshake.rs index d9e280bfe..af67f7ddf 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -12,23 +12,39 @@ use network::NetworkError; #[derive(PartialEq, Eq, Debug)] enum HandshakeState { + /// Just created New, + /// Waiting for auth packet ReadingAuth, + /// Waiting for ack packet ReadingAck, + /// Ready to start a session StartSession, } +/// RLPx protocol handhake. See https://github.com/ethereum/devp2p/blob/master/rlpx.md#encrypted-handshake pub struct Handshake { + /// Remote node public key pub id: NodeId, + /// Underlying connection pub connection: Connection, + /// Handshake state state: HandshakeState, + /// Outgoing or incoming connection pub originated: bool, + /// Disconnect timeout idle_timeout: Option, + /// ECDH ephemeral pub ecdhe: KeyPair, + /// Connection nonce pub nonce: H256, + /// Handshake public key pub remote_public: Public, + /// Remote connection nonce. pub remote_nonce: H256, + /// A copy of received encryped auth packet pub auth_cipher: Bytes, + /// A copy of received encryped ack packet pub ack_cipher: Bytes } @@ -36,6 +52,7 @@ const AUTH_PACKET_SIZE: usize = 307; const ACK_PACKET_SIZE: usize = 210; impl Handshake { + /// Create a new handshake object pub fn new(token: Token, id: &NodeId, socket: TcpStream, nonce: &H256) -> Result { Ok(Handshake { id: id.clone(), @@ -52,6 +69,7 @@ impl Handshake { }) } + /// Start a handhsake pub fn start(&mut self, host: &HostInfo, originated: bool) -> Result<(), UtilError> { self.originated = originated; if originated { @@ -64,10 +82,12 @@ impl Handshake { Ok(()) } + /// Check if handshake is complete pub fn done(&self) -> bool { self.state == HandshakeState::StartSession } + /// Readable IO handler. Drives the state change. pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); match self.state { @@ -97,6 +117,7 @@ impl Handshake { Ok(()) } + /// Writabe IO handler. pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); @@ -106,6 +127,7 @@ impl Handshake { Ok(()) } + /// Register the IO handler with the event loop pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); @@ -113,6 +135,7 @@ impl Handshake { Ok(()) } + /// Parse, validate and confirm auth message fn read_auth(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); assert!(data.len() == AUTH_PACKET_SIZE); @@ -134,6 +157,7 @@ impl Handshake { self.write_ack() } + /// Parse and validate ack message fn read_ack(&mut self, host: &HostInfo, data: &[u8]) -> Result<(), UtilError> { trace!(target:"net", "Received handshake auth to {:?}", self.connection.socket.peer_addr()); assert!(data.len() == ACK_PACKET_SIZE); @@ -144,6 +168,7 @@ impl Handshake { Ok(()) } + /// Sends auth message fn write_auth(&mut self, host: &HostInfo) -> Result<(), UtilError> { trace!(target:"net", "Sending handshake auth to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; /*Signature::SIZE*/ 65 + /*H256::SIZE*/ 32 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32 + 1]; //TODO: use associated constants @@ -170,6 +195,7 @@ impl Handshake { Ok(()) } + /// Sends ack message fn write_ack(&mut self) -> Result<(), UtilError> { trace!(target:"net", "Sending handshake ack to {:?}", self.connection.socket.peer_addr()); let mut data = [0u8; 1 + /*Public::SIZE*/ 64 + /*H256::SIZE*/ 32]; //TODO: use associated constants diff --git a/src/network/host.rs b/src/network/host.rs index f48b9eb2d..11dc491b2 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -22,7 +22,9 @@ const MAX_CONNECTIONS: usize = 1024; const MAX_USER_TIMERS: usize = 32; const IDEAL_PEERS: u32 = 10; +/// Node public key pub type NodeId = H512; +/// IO Timer id pub type TimerToken = usize; #[derive(Debug)] @@ -47,13 +49,18 @@ impl NetworkConfiguration { } #[derive(Debug)] +/// Noe address info pub struct NodeEndpoint { + /// IP(V4 or V6) address address: SocketAddr, + /// Address as string (can be host name). address_str: String, + /// Conneciton port. udp_port: u16 } impl NodeEndpoint { + /// Create endpoint from string. Performs name resolution if given a host name. fn from_str(s: &str) -> Result { let address = s.to_socket_addrs().map(|mut i| i.next()); match address { @@ -124,39 +131,52 @@ const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; const USER_TIMER: usize = LAST_CONNECTION; const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; +/// Protocol handler level packet id pub type PacketId = u8; +/// Protocol / handler id pub type ProtocolId = &'static str; +/// Messages used to communitate with the event loop from other threads. pub enum HostMessage { + /// Shutdown the event loop Shutdown, + /// Register a new protocol handler. AddHandler { handler: Box, protocol: ProtocolId, versions: Vec, }, + /// Send data over the network. Send { peer: PeerId, packet_id: PacketId, protocol: ProtocolId, data: Vec, }, + /// Broadcast a message across the protocol handlers. UserMessage(UserMessage), } +/// Id for broadcast message pub type UserMessageId = u32; +/// User pub struct UserMessage { + /// ID of a protocol pub protocol: ProtocolId, pub id: UserMessageId, pub data: Option>, } +/// Local (temporary) peer session ID. pub type PeerId = usize; #[derive(Debug, PartialEq, Eq)] +/// Protocol info pub struct CapabilityInfo { pub protocol: ProtocolId, pub version: u8, + /// Total number of packet IDs this protocol support. pub packet_count: u8, } @@ -169,7 +189,7 @@ impl Encodable for CapabilityInfo { } } -/// IO access point +/// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. pub struct HostIo<'s> { protocol: ProtocolId, connections: &'s mut Slab, @@ -179,6 +199,7 @@ pub struct HostIo<'s> { } impl<'s> HostIo<'s> { + /// Create a new IO access point. Takes references to all the data that can be updated within the IO handler. fn new(protocol: ProtocolId, session: Option, event_loop: &'s mut EventLoop, connections: &'s mut Slab, timers: &'s mut Slab) -> HostIo<'s> { HostIo { protocol: protocol, @@ -252,24 +273,36 @@ struct UserTimer { delay: u64, } +/// Shared host information pub struct HostInfo { + /// Our private and public keys. keys: KeyPair, + /// Current network configuration config: NetworkConfiguration, + /// Connection nonce. nonce: H256, + /// RLPx protocol version pub protocol_version: u32, + /// Client identifier pub client_version: String, + /// TCP connection port. pub listen_port: u16, + /// Registered capabilities (handlers) pub capabilities: Vec } impl HostInfo { + /// Returns public key pub fn id(&self) -> &NodeId { self.keys.public() } + /// Returns secret key pub fn secret(&self) -> &Secret { self.keys.secret() } + + /// Increments and returns connection nonce. pub fn next_nonce(&mut self) -> H256 { self.nonce = self.nonce.sha3(); return self.nonce.clone(); @@ -281,6 +314,7 @@ enum ConnectionEntry { Session(Session) } +/// Root IO handler. Manages protocol handlers, IO timers and network connections. pub struct Host { info: HostInfo, _udp_socket: UdpSocket, @@ -293,6 +327,7 @@ pub struct Host { } impl Host { + /// Creates a new instance and registers it with the event loop. pub fn start(event_loop: &mut EventLoop) -> Result<(), UtilError> { let config = NetworkConfiguration::new(); /* @@ -457,7 +492,7 @@ impl Host { fn accept(&mut self, _event_loop: &mut EventLoop) { - warn!(target: "net", "accept"); + trace!(target: "net", "accept"); } fn connection_writable(&mut self, token: Token, event_loop: &mut EventLoop) { diff --git a/src/network/session.rs b/src/network/session.rs index 141898e24..96390d366 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -7,27 +7,44 @@ use error::*; use network::{NetworkError, DisconnectReason}; use network::host::*; +/// Peer session over encrypted connection. +/// When created waits for Hello packet exchange and signals ready state. +/// Sends and receives protocol packets and handles basic packes such as ping/pong and disconnect. pub struct Session { + /// Shared session information pub info: SessionInfo, + /// Underlying connection connection: EncryptedConnection, + /// Session ready flag. Set after successfull Hello packet exchange had_hello: bool, } +/// Structure used to report various session events. pub enum SessionData { None, + /// Session is ready to send/receive packets. Ready, + /// A packet has been received Packet { + /// Packet data data: Vec, + /// Packet protocol ID protocol: &'static str, + /// Zero based packet ID packet_id: u8, }, } +/// Shared session information pub struct SessionInfo { + /// Peer public key pub id: NodeId, + /// Peer client ID pub client_version: String, + /// Peer RLPx protocol version pub protocol_version: u32, - pub capabilities: Vec, + /// Peer protocol capabilities + capabilities: Vec, } #[derive(Debug, PartialEq, Eq)] @@ -48,7 +65,7 @@ impl Decodable for PeerCapabilityInfo { } #[derive(Debug, PartialEq, Eq)] -pub struct SessionCapabilityInfo { +struct SessionCapabilityInfo { pub protocol: &'static str, pub version: u8, pub packet_count: u8, @@ -65,6 +82,7 @@ const PACKET_USER: u8 = 0x10; const PACKET_LAST: u8 = 0x7f; impl Session { + /// Create a new session out of comepleted handshake. Consumes handshake object. pub fn new(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); @@ -84,10 +102,12 @@ impl Session { Ok(session) } + /// Check if session is ready to send/receive data pub fn is_ready(&self) -> bool { self.had_hello } + /// Readable IO handler. Returns packet data if available. pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { match try!(self.connection.readable(event_loop)) { Some(data) => Ok(try!(self.read_packet(data, host))), @@ -95,18 +115,22 @@ impl Session { } } + /// Writable IO handler. Sends pending packets. pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { self.connection.writable(event_loop) } + /// Checks if peer supports given capability pub fn have_capability(&self, protocol: &str) -> bool { self.info.capabilities.iter().any(|c| c.protocol == protocol) } + /// Update registration with the event loop. Should be called at the end of the IO handler. pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.connection.reregister(event_loop) } + /// Send a protocol packet to peer. pub fn send_packet(&mut self, protocol: &str, packet_id: u8, data: &[u8]) -> Result<(), UtilError> { let mut i = 0usize; while protocol != self.info.capabilities[i].protocol { From c269cb5c85585679aac1b4d7d61bf12c36f3fee0 Mon Sep 17 00:00:00 2001 From: arkpar Date: Mon, 11 Jan 2016 11:52:18 +0100 Subject: [PATCH 315/379] Added sync to std uses --- src/standard.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/standard.rs b/src/standard.rs index b591220f3..dfdf4488b 100644 --- a/src/standard.rs +++ b/src/standard.rs @@ -9,6 +9,7 @@ pub use std::io::{Read,Write}; pub use std::hash::{Hash, Hasher}; pub use std::error::Error as StdError; +pub use std::sync::*; pub use std::ops::*; pub use std::cmp::*; pub use std::cell::*; From c104009ef65e34d473dc319bcf2b200787410ea9 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 11 Jan 2016 17:38:28 +0100 Subject: [PATCH 316/379] Additional standard.rs. --- src/standard.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/standard.rs b/src/standard.rs index dfdf4488b..07865f8ec 100644 --- a/src/standard.rs +++ b/src/standard.rs @@ -2,6 +2,10 @@ pub use std::io; pub use std::str; pub use std::fmt; pub use std::slice; +pub use std::cmp; +pub use std::ptr; +pub use std::result; +pub use std::option; pub use std::path::Path; pub use std::str::{FromStr}; From 1c48e34d69bf7182b5abec34b7851bf61753e2cb Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 11 Jan 2016 18:01:54 +0100 Subject: [PATCH 317/379] Additional mem use. --- src/standard.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/standard.rs b/src/standard.rs index 07865f8ec..7298dc232 100644 --- a/src/standard.rs +++ b/src/standard.rs @@ -6,6 +6,7 @@ pub use std::cmp; pub use std::ptr; pub use std::result; pub use std::option; +pub use std::mem; pub use std::path::Path; pub use std::str::{FromStr}; From 65ab52405394cbfc9e4e516360513335f4ba7dfc Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Mon, 11 Jan 2016 23:59:33 +0100 Subject: [PATCH 318/379] Avoid panic on invalid uint data. --- src/bytes.rs | 6 +++++- src/lib.rs | 1 + src/uint.rs | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/bytes.rs b/src/bytes.rs index 8d33a4108..98ab4c2d7 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -305,7 +305,11 @@ macro_rules! impl_uint_from_bytes { ($name: ident) => { impl FromBytes for $name { fn from_bytes(bytes: &[u8]) -> FromBytesResult<$name> { - Ok($name::from(bytes)) + if bytes.len() <= $name::SIZE { + Ok($name::from(bytes)) + } else { + Err(FromBytesError::DataIsTooLong) + } } } } diff --git a/src/lib.rs b/src/lib.rs index de2cdbdf7..b856431cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![feature(op_assign_traits)] +#![feature(associated_consts)] //! Ethcore-util library //! //! ### Rust version: diff --git a/src/uint.rs b/src/uint.rs index 6fc8d19e1..b7411fd30 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -50,6 +50,8 @@ macro_rules! construct_uint { pub struct $name(pub [u64; $n_words]); impl $name { + pub const SIZE: usize = $n_words * 8; + /// Conversion to u32 #[inline] pub fn low_u32(&self) -> u32 { From 48fbf24e21477a941656a90bc010985d319e0df3 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 12 Jan 2016 00:10:47 +0100 Subject: [PATCH 319/379] Avoid panicking on oversize u64. --- src/bytes.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bytes.rs b/src/bytes.rs index 98ab4c2d7..faab23be3 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -264,7 +264,7 @@ impl FromBytes for u64 { fn from_bytes(bytes: &[u8]) -> FromBytesResult { match bytes.len() { 0 => Ok(0), - l => { + l @ 1...8 => { let mut res = 0u64; for i in 0..l { let shift = (l - 1 - i) * 8; @@ -272,6 +272,7 @@ impl FromBytes for u64 { } Ok(res) } + _ => Err(FromBytesError::DataIsTooLong) } } } From e2de777c300f42c9aae93bcfe9126cf681621a72 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 12 Jan 2016 00:55:42 +0100 Subject: [PATCH 320/379] Fix panic in crypto, avoid incorrect casting in bytes. --- src/bytes.rs | 46 ++++++++++++++++++++------------------------ src/crypto.rs | 2 +- src/rlp/rlperrors.rs | 2 +- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index faab23be3..21d28e87c 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -34,6 +34,7 @@ //! } //! ``` +use std::mem; use std::fmt; use std::slice; use std::cmp::Ordering; @@ -260,19 +261,23 @@ impl FromBytes for String { } } -impl FromBytes for u64 { - fn from_bytes(bytes: &[u8]) -> FromBytesResult { - match bytes.len() { - 0 => Ok(0), - l @ 1...8 => { - let mut res = 0u64; - for i in 0..l { - let shift = (l - 1 - i) * 8; - res = res + ((bytes[i] as u64) << shift); +macro_rules! impl_uint_from_bytes { + ($to: ident) => { + impl FromBytes for $to { + fn from_bytes(bytes: &[u8]) -> FromBytesResult<$to> { + match bytes.len() { + 0 => Ok(0), + l if l <= mem::size_of::<$to>() => { + let mut res = 0 as $to; + for i in 0..l { + let shift = (l - 1 - i) * 8; + res = res + ((bytes[i] as $to) << shift); + } + Ok(res) + } + _ => Err(FromBytesError::DataIsTooLong) } - Ok(res) } - _ => Err(FromBytesError::DataIsTooLong) } } } @@ -287,20 +292,11 @@ impl FromBytes for bool { } } - -macro_rules! impl_map_from_bytes { - ($from: ident, $to: ident) => { - impl FromBytes for $from { - fn from_bytes(bytes: &[u8]) -> FromBytesResult<$from> { - $to::from_bytes(bytes).map(| x | { x as $from }) - } - } - } -} - -impl_map_from_bytes!(usize, u64); -impl_map_from_bytes!(u16, u64); -impl_map_from_bytes!(u32, u64); +//impl_uint_from_bytes!(u8); +impl_uint_from_bytes!(u16); +impl_uint_from_bytes!(u32); +impl_uint_from_bytes!(u64); +impl_uint_from_bytes!(usize); macro_rules! impl_uint_from_bytes { ($name: ident) => { diff --git a/src/crypto.rs b/src/crypto.rs index a80045e55..d13a53f53 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -37,7 +37,7 @@ impl From<::secp256k1::Error> for CryptoError { ::secp256k1::Error::InvalidPublicKey => CryptoError::InvalidPublic, ::secp256k1::Error::InvalidSignature => CryptoError::InvalidSignature, ::secp256k1::Error::InvalidSecretKey => CryptoError::InvalidSecret, - _ => panic!("Crypto error: {:?}", e), + _ => CryptoError::InvalidSignature, } } } diff --git a/src/rlp/rlperrors.rs b/src/rlp/rlperrors.rs index 4ee41a2ce..27b3082b5 100644 --- a/src/rlp/rlperrors.rs +++ b/src/rlp/rlperrors.rs @@ -8,7 +8,7 @@ pub enum DecoderError { RlpIsTooShort, RlpExpectedToBeList, RlpExpectedToBeData, - RlpIncorrectListLen + RlpIncorrectListLen, } impl StdError for DecoderError { From 982063e1ac8db74b263960d5cfaba0428064acfe Mon Sep 17 00:00:00 2001 From: arkpar Date: Tue, 12 Jan 2016 17:33:40 +0100 Subject: [PATCH 321/379] Started IO service refactoring --- src/io/mod.rs | 68 +++++++++++++++ src/io/service.rs | 207 ++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 3 files changed, 276 insertions(+) create mode 100644 src/io/mod.rs create mode 100644 src/io/service.rs diff --git a/src/io/mod.rs b/src/io/mod.rs new file mode 100644 index 000000000..e8562e64a --- /dev/null +++ b/src/io/mod.rs @@ -0,0 +1,68 @@ +/// General IO module. +/// +/// Example usage for craeting a network service and adding an IO handler: +/// +/// ```rust +/// extern crate ethcore_util as util; +/// use util::network::*; +/// +/// struct MyHandler; +/// +/// impl ProtocolHandler for MyHandler { +/// fn initialize(&mut self, io: &mut HandlerIo) { +/// io.register_timer(1000); +/// } +/// +/// fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { +/// println!("Received {} ({} bytes) from {}", packet_id, data.len(), peer); +/// } +/// +/// fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { +/// println!("Connected {}", peer); +/// } +/// +/// fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { +/// println!("Disconnected {}", peer); +/// } +/// +/// fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { +/// println!("Timeout {}", timer); +/// } +/// +/// fn message(&mut self, io: &mut HandlerIo, message: &Message) { +/// println!("Message {}:{}", message.protocol, message.id); +/// } +/// } +/// +/// fn main () { +/// let mut service = NetworkService::start().expect("Error creating network service"); +/// service.register_protocol(Box::new(MyHandler), "myproto", &[1u8]); +/// +/// // Wait for quit condition +/// // ... +/// // Drop the service +/// } +/// ``` +extern crate mio; +mod service; + +#[derive(Debug)] +pub enum IoError { + Mio(::std::io::Error), +} + +impl From<::mio::NotifyError>> for IoError { + fn from(_err: ::mio::NotifyError>) -> IoError { + IoError::Mio(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) + } +} + +pub type TimerToken = service::TimerToken; +pub type StreamToken = service::StreamToken; +pub type IoContext<'s, M> = service::IoContext<'s, M>; +pub type Message = service::UserMessage; +pub type IoService = service::IoService; +pub type IoHandler = service::IoHandler; + + + diff --git a/src/io/service.rs b/src/io/service.rs new file mode 100644 index 000000000..4ecc34723 --- /dev/null +++ b/src/io/service.rs @@ -0,0 +1,207 @@ +use std::thread::{self, JoinHandle}; +use mio::*; +use mio::util::{Slab}; +use hash::*; +use rlp::*; +use error::*; +use io::IoError; + +/// Generic IO handler. +/// All the handler function are called from within IO event loop. +pub trait IoHandler: Send where M: Send + 'static { + /// Initialize the hadler + fn initialize(&mut self, _io: &mut IoContext) {} + /// Timer function called after a timeout created with `HandlerIo::timeout`. + fn timeout(&mut self, _io: &mut IoContext, _timer: TimerToken) {} + /// Called when a broadcasted message is received. The message can only be sent from a different IO handler. + fn message(&mut self, _io: &mut IoContext, _message: &M) {} + /// Called when an IO stream gets closed + fn stream_hup(&mut self, _io: &mut IoContext, _stream: StreamToken) {} + /// Called when an IO stream can be read from + fn stream_readable(&mut self, _io: &mut IoContext, _stream: StreamToken) {} + /// Called when an IO stream can be written to + fn stream_writable(&mut self, _io: &mut IoContext, _stream: StreamToken) {} +} + +pub type TimerToken = usize; +pub type StreamToken = usize; + +// Tokens +const MAX_USER_TIMERS: usize = 32; +const USER_TIMER: usize = 0; +const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; + +/// Messages used to communicate with the event loop from other threads. +pub enum IoMessage { + /// Shutdown the event loop + Shutdown, + /// Register a new protocol handler. + AddHandler { + handler: Box+Send>, + }, + /// Broadcast a message across all protocol handlers. + UserMessage(UserMessage), +} + +/// User +pub struct UserMessage { + pub data: M, +} + +/// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. +pub struct IoContext<'s, M> where M: Send + 'static { + timers: &'s mut Slab, + event_loop: &'s mut EventLoop>, +} + +impl<'s, M> IoContext<'s, M> where M: Send + 'static { + /// Create a new IO access point. Takes references to all the data that can be updated within the IO handler. + fn new(event_loop: &'s mut EventLoop>, timers: &'s mut Slab) -> IoContext<'s, M> { + IoContext { + event_loop: event_loop, + timers: timers, + } + } + + /// Register a new IO timer. Returns a new timer token. 'IoHandler::timeout' will be called with the token. + pub fn register_timer(&mut self, ms: u64) -> Result{ + match self.timers.insert(UserTimer { + delay: ms, + }) { + Ok(token) => { + self.event_loop.timeout_ms(token, ms).expect("Error registering user timer"); + Ok(token.as_usize()) + }, + _ => { panic!("Max timers reached") } + } + } + + /// Broadcast a message to other IO clients + pub fn message(&mut self, message: M) { + match self.event_loop.channel().send(IoMessage::UserMessage(UserMessage { + data: message + })) { + Ok(_) => {} + Err(e) => { panic!("Error sending io message {:?}", e); } + } + } +} + +struct UserTimer { + delay: u64, +} + +/// Root IO handler. Manages user handlers, messages and IO timers. +pub struct IoManager where M: Send { + timers: Slab, + handlers: Vec>>, +} + +impl IoManager where M: Send + 'static { + /// Creates a new instance and registers it with the event loop. + pub fn start(event_loop: &mut EventLoop>) -> Result<(), UtilError> { + let mut io = IoManager { + timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), + handlers: Vec::new(), + }; + try!(event_loop.run(&mut io)); + Ok(()) + } +} + +impl Handler for IoManager where M: Send + 'static { + type Timeout = Token; + type Message = IoMessage; + + fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { + if events.is_hup() { + for h in self.handlers.iter_mut() { + h.stream_hup(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + } + } + else if events.is_readable() { + for h in self.handlers.iter_mut() { + h.stream_readable(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + } + } + else if events.is_writable() { + for h in self.handlers.iter_mut() { + h.stream_writable(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + } + } + } + + fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { + match token.as_usize() { + USER_TIMER ... LAST_USER_TIMER => { + let delay = { + let timer = self.timers.get_mut(token).expect("Unknown user timer token"); + timer.delay + }; + for h in self.handlers.iter_mut() { + h.timeout(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + } + event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); + } + _ => { // Just pass the event down. IoHandler is supposed to re-register it if required. + for h in self.handlers.iter_mut() { + h.timeout(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + } + } + } + } + + fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { + match msg { + IoMessage::Shutdown => event_loop.shutdown(), + IoMessage::AddHandler { + handler, + } => { + self.handlers.push(handler); + }, + IoMessage::UserMessage(message) => { + for h in self.handlers.iter_mut() { + h.message(&mut IoContext::new(event_loop, &mut self.timers), &message.data); + } + } + } + } +} + +/// General IO Service. Starts an event loop and dispatches IO requests. +/// 'M' is a notification message type +pub struct IoService where M: Send + 'static { + thread: Option>, + host_channel: Sender> +} + +impl IoService where M: Send + 'static { + /// Starts IO event loop + pub fn start() -> Result, UtilError> { + let mut event_loop = EventLoop::new().unwrap(); + let channel = event_loop.channel(); + let thread = thread::spawn(move || { + IoManager::::start(&mut event_loop).unwrap(); //TODO: + }); + Ok(IoService { + thread: Some(thread), + host_channel: channel + }) + } + + /// Regiter a IO hadnler with the event loop. + pub fn register_handler(&mut self, handler: Box+Send>) -> Result<(), IoError> { + try!(self.host_channel.send(IoMessage::AddHandler { + handler: handler, + })); + Ok(()) + } +} + +impl Drop for IoService where M: Send { + fn drop(&mut self) { + self.host_channel.send(IoMessage::Shutdown).unwrap(); + self.thread.take().unwrap().join().unwrap(); + } +} + diff --git a/src/lib.rs b/src/lib.rs index b856431cd..acd96ee5f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -71,6 +71,7 @@ pub mod nibbleslice; pub mod heapsizeof; pub mod squeeze; pub mod semantic_version; +pub mod io; pub mod network; pub use common::*; From 6b1eb943e21b01725798dbdb4d30ce6ec025fec6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 12 Jan 2016 17:40:55 +0100 Subject: [PATCH 322/379] Add U512 type and a few conversions. --- src/bytes.rs | 20 ++++++++-------- src/crypto.rs | 7 ++++++ src/uint.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 21d28e87c..8c56d6bf3 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -109,25 +109,25 @@ impl BytesConvertable for Vec { } macro_rules! impl_bytes_convertable_for_array { - ($zero: expr) => (); - ($len: expr, $($idx: expr),*) => { - impl BytesConvertable for [u8; $len] { - fn bytes(&self) -> &[u8] { self } - } - impl_bytes_convertable_for_array! { $($idx),* } - } + ($zero: expr) => (); + ($len: expr, $($idx: expr),*) => { + impl BytesConvertable for [u8; $len] { + fn bytes(&self) -> &[u8] { self } + } + impl_bytes_convertable_for_array! { $($idx),* } + } } // -1 at the end is not expanded impl_bytes_convertable_for_array! { - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -1 + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, -1 } #[test] fn bytes_convertable() { assert_eq!(vec![0x12u8, 0x34].bytes(), &[0x12u8, 0x34]); - assert_eq!([0u8; 0].bytes(), &[]); + assert_eq!([0u8; 0].bytes(), &[]); } /// Converts given type to its shortest representation in bytes diff --git a/src/crypto.rs b/src/crypto.rs index d13a53f53..9a27c68ae 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -1,4 +1,6 @@ use hash::*; +use bytes::*; +use uint::*; use secp256k1::{key, Secp256k1}; use rand::os::OsRng; @@ -19,6 +21,11 @@ impl Signature { ret[64] = v; ret } + + /// Convert transaction to R, S and V components. + pub fn to_rsv(&self) -> (U256, U256, u8) { + (U256::from(&self.as_slice()[0..32]), U256::from(&self.as_slice()[32..64]), self[64]) + } } #[derive(Debug)] diff --git a/src/uint.rs b/src/uint.rs index b7411fd30..6fefbd50d 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -462,9 +462,73 @@ macro_rules! construct_uint { ); } +construct_uint!(U512, 8); construct_uint!(U256, 4); construct_uint!(U128, 2); +impl From for U512 { + fn from(value: U256) -> U512 { + let U256(ref arr) = value; + let mut ret = [0; 8]; + ret[0] = arr[0]; + ret[1] = arr[1]; + ret[2] = arr[2]; + ret[3] = arr[3]; + U512(ret) + } +} + +impl From for U256 { + fn from(value: U512) -> U256 { + let U512(ref arr) = value; + if arr[4] | arr[5] | arr[6] | arr[7] != 0 { + panic!("Overflow"); + } + let mut ret = [0; 4]; + ret[0] = arr[0]; + ret[1] = arr[1]; + ret[2] = arr[2]; + ret[3] = arr[3]; + U256(ret) + } +} + +impl From for U128 { + fn from(value: U256) -> U128 { + let U256(ref arr) = value; + if arr[2] | arr[3] != 0 { + panic!("Overflow"); + } + let mut ret = [0; 2]; + ret[0] = arr[0]; + ret[1] = arr[1]; + U128(ret) + } +} + +impl From for U128 { + fn from(value: U512) -> U128 { + let U512(ref arr) = value; + if arr[2] | arr[3] | arr[4] | arr[5] | arr[6] | arr[7] != 0 { + panic!("Overflow"); + } + let mut ret = [0; 2]; + ret[0] = arr[0]; + ret[1] = arr[1]; + U128(ret) + } +} + +impl From for U512 { + fn from(value: U128) -> U512 { + let U128(ref arr) = value; + let mut ret = [0; 8]; + ret[0] = arr[0]; + ret[1] = arr[1]; + U512(ret) + } +} + impl From for U256 { fn from(value: U128) -> U256 { let U128(ref arr) = value; From 635bf284b4e63552fdc6037b15cdff74fa81fbd5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Tue, 12 Jan 2016 23:44:30 +0100 Subject: [PATCH 323/379] Fix RLP on consensus. Add JSON helpers. --- src/crypto.rs | 13 ++++++++ src/json_aid.rs | 65 ++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 2 ++ src/rlp/rlperrors.rs | 3 ++ src/rlp/untrusted_rlp.rs | 10 ++++++- 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/json_aid.rs diff --git a/src/crypto.rs b/src/crypto.rs index 9a27c68ae..35bb3b7eb 100644 --- a/src/crypto.rs +++ b/src/crypto.rs @@ -120,6 +120,8 @@ impl KeyPair { pub mod ec { use hash::*; + use uint::*; + use standard::*; use crypto::*; use crypto::{self}; @@ -136,6 +138,7 @@ pub mod ec { } /// Returns siganture of message hash. pub fn sign(secret: &Secret, message: &H256) -> Result { + // TODO: allow creation of only low-s signatures. use secp256k1::*; let context = Secp256k1::new(); let sec: &key::SecretKey = unsafe { ::std::mem::transmute(secret) }; @@ -165,6 +168,16 @@ pub mod ec { } } + /// Check if this is a "low" signature. + pub fn is_low(sig: &Signature) -> bool { + H256::from_slice(&sig[32..64]) <= h256_from_hex("7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0") + } + + /// Check if this is a "low" signature. + pub fn is_low_s(s: &U256) -> bool { + s <= &U256::from_str("7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0").unwrap() + } + /// Check if each component of the signature is in range. pub fn is_valid(sig: &Signature) -> bool { sig[64] <= 1 && diff --git a/src/json_aid.rs b/src/json_aid.rs new file mode 100644 index 000000000..ccfdee490 --- /dev/null +++ b/src/json_aid.rs @@ -0,0 +1,65 @@ +use common::*; + +pub fn clean(s: &str) -> &str { + if s.len() >= 2 && &s[0..2] == "0x" { + &s[2..] + } else { + s + } +} + +pub fn bytes_from_json(json: &Json) -> Bytes { + let s = json.as_string().unwrap_or(""); + if s.len() % 2 == 1 { + FromHex::from_hex(&("0".to_string() + &(clean(s).to_string()))[..]).unwrap_or(vec![]) + } else { + FromHex::from_hex(clean(s)).unwrap_or(vec![]) + } +} + +pub fn address_from_json(json: &Json) -> Address { + let s = json.as_string().unwrap_or("0000000000000000000000000000000000000000"); + if s.len() % 2 == 1 { + address_from_hex(&("0".to_string() + &(clean(s).to_string()))[..]) + } else { + address_from_hex(clean(s)) + } +} + +pub fn h256_from_json(json: &Json) -> H256 { + let s = json.as_string().unwrap_or("0000000000000000000000000000000000000000000000000000000000000000"); + if s.len() % 2 == 1 { + h256_from_hex(&("0".to_string() + &(clean(s).to_string()))[..]) + } else { + h256_from_hex(clean(s)) + } +} + +pub fn u256_from_json(json: &Json) -> U256 { + let s = json.as_string().unwrap_or(""); + if s.len() >= 2 && &s[0..2] == "0x" { + U256::from_str(&s[2..]).unwrap_or(U256::from(0)) + } else { + U256::from_dec_str(s).unwrap_or(U256::from(0)) + } +} + +pub fn usize_from_json(json: &Json) -> usize { + u256_from_json(json).low_u64() as usize +} + +pub fn u64_from_json(json: &Json) -> u64 { + u256_from_json(json).low_u64() +} + +pub fn u32_from_json(json: &Json) -> u32 { + u256_from_json(json).low_u32() +} + +pub fn u16_from_json(json: &Json) -> u16 { + u256_from_json(json).low_u32() as u16 +} + +pub fn u8_from_json(json: &Json) -> u8 { + u256_from_json(json).low_u32() as u8 +} diff --git a/src/lib.rs b/src/lib.rs index b856431cd..7bfe147f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -57,6 +57,7 @@ pub mod hash; pub mod uint; pub mod bytes; pub mod rlp; +pub mod json_aid; pub mod vector; pub mod sha3; pub mod hashdb; @@ -74,6 +75,7 @@ pub mod semantic_version; pub mod network; pub use common::*; +pub use json_aid::*; pub use rlp::*; pub use hashdb::*; pub use memorydb::*; diff --git a/src/rlp/rlperrors.rs b/src/rlp/rlperrors.rs index 27b3082b5..9ea470abf 100644 --- a/src/rlp/rlperrors.rs +++ b/src/rlp/rlperrors.rs @@ -9,6 +9,9 @@ pub enum DecoderError { RlpExpectedToBeList, RlpExpectedToBeData, RlpIncorrectListLen, + RlpDataLenWithZeroPrefix, + RlpListLenWithZeroPrefix, + RlpInvalidIndirection, } impl StdError for DecoderError { diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index 452a198bb..c0e7c4cf7 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -269,6 +269,7 @@ impl<'a> BasicDecoder<'a> { Some(l @ 0xb8...0xbf) => { let len_of_len = l as usize - 0xb7; let header_len = 1 + len_of_len; + if bytes[1] == 0 { return Err(DecoderError::RlpDataLenWithZeroPrefix); } let value_len = try!(usize::from_bytes(&bytes[1..header_len])); PayloadInfo::new(header_len, value_len) } @@ -277,6 +278,7 @@ impl<'a> BasicDecoder<'a> { let len_of_len = l as usize - 0xf7; let header_len = 1 + len_of_len; let value_len = try!(usize::from_bytes(&bytes[1..header_len])); + if bytes[1] == 0 { return Err(DecoderError::RlpListLenWithZeroPrefix); } PayloadInfo::new(header_len, value_len) }, // we cant reach this place, but rust requires _ to be implemented @@ -302,7 +304,13 @@ impl<'a> Decoder for BasicDecoder<'a> { // single byt value Some(l @ 0...0x7f) => Ok(try!(f(&[l]))), // 0-55 bytes - Some(l @ 0x80...0xb7) => Ok(try!(f(&bytes[1..(1 + l as usize - 0x80)]))), + Some(l @ 0x80...0xb7) => { + let d = &bytes[1..(1 + l as usize - 0x80)]; + if l == 0x81 && d[0] < 0x80 { + return Err(DecoderError::RlpInvalidIndirection); + } + Ok(try!(f(d))) + }, // longer than 55 bytes Some(l @ 0xb8...0xbf) => { let len_of_len = l as usize - 0xb7; From 6447f9b90bde28a9851605730899e661ace850ad Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 01:18:33 +0100 Subject: [PATCH 324/379] Additional helper function. --- src/json_aid.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/json_aid.rs b/src/json_aid.rs index ccfdee490..a382c61ec 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -35,8 +35,7 @@ pub fn h256_from_json(json: &Json) -> H256 { } } -pub fn u256_from_json(json: &Json) -> U256 { - let s = json.as_string().unwrap_or(""); +pub fn u256_from_hex(s: &str) -> U256 { if s.len() >= 2 && &s[0..2] == "0x" { U256::from_str(&s[2..]).unwrap_or(U256::from(0)) } else { @@ -44,6 +43,10 @@ pub fn u256_from_json(json: &Json) -> U256 { } } +pub fn u256_from_json(json: &Json) -> U256 { + u256_from_hex(json.as_string().unwrap_or("")) +} + pub fn usize_from_json(json: &Json) -> usize { u256_from_json(json).low_u64() as usize } From c98f73c5c9953bb122322dc5a3b64a7021aa5c41 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 13 Jan 2016 11:31:37 +0100 Subject: [PATCH 325/379] Finished splitting IoService and NetworkService --- src/error.rs | 10 +- src/io/mod.rs | 23 +- src/io/service.rs | 58 ++--- src/network/connection.rs | 17 +- src/network/discovery.rs | 6 +- src/network/handshake.rs | 11 +- src/network/host.rs | 470 +++++++++++++++----------------------- src/network/mod.rs | 84 ++----- src/network/service.rs | 43 ++-- src/network/session.rs | 11 +- 10 files changed, 296 insertions(+), 437 deletions(-) diff --git a/src/error.rs b/src/error.rs index 17b65ceec..04f7b96ce 100644 --- a/src/error.rs +++ b/src/error.rs @@ -3,6 +3,7 @@ use rustc_serialize::hex::FromHexError; use network::NetworkError; use rlp::DecoderError; +use io; #[derive(Debug)] pub enum BaseDataError { @@ -13,7 +14,8 @@ pub enum BaseDataError { /// General error type which should be capable of representing all errors in ethcore. pub enum UtilError { Crypto(::crypto::CryptoError), - Io(::std::io::Error), + StdIo(::std::io::Error), + Io(io::IoError), AddressParse(::std::net::AddrParseError), AddressResolve(Option<::std::io::Error>), FromHex(FromHexError), @@ -43,6 +45,12 @@ impl From for UtilError { impl From<::std::io::Error> for UtilError { fn from(err: ::std::io::Error) -> UtilError { + UtilError::StdIo(err) + } +} + +impl From for UtilError { + fn from(err: io::IoError) -> UtilError { UtilError::Io(err) } } diff --git a/src/io/mod.rs b/src/io/mod.rs index e8562e64a..72906b6f4 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -51,18 +51,35 @@ pub enum IoError { Mio(::std::io::Error), } -impl From<::mio::NotifyError>> for IoError { +impl From<::mio::NotifyError>> for IoError where M: Send { fn from(_err: ::mio::NotifyError>) -> IoError { IoError::Mio(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) } } +/// Generic IO handler. +/// All the handler function are called from within IO event loop. +/// `Message` type is used as notification data +pub trait IoHandler: Send where Message: Send + 'static { + /// Initialize the handler + fn initialize(&mut self, _io: IoContext) {} + /// Timer function called after a timeout created with `HandlerIo::timeout`. + fn timeout(&mut self, _io: IoContext, _timer: TimerToken) {} + /// Called when a broadcasted message is received. The message can only be sent from a different IO handler. + fn message(&mut self, _io: IoContext, _message: &mut Message) {} // TODO: make message immutable and provide internal channel for adding network handler + /// Called when an IO stream gets closed + fn stream_hup(&mut self, _io: IoContext, _stream: StreamToken) {} + /// Called when an IO stream can be read from + fn stream_readable(&mut self, _io: IoContext, _stream: StreamToken) {} + /// Called when an IO stream can be written to + fn stream_writable(&mut self, _io: IoContext, _stream: StreamToken) {} +} + pub type TimerToken = service::TimerToken; pub type StreamToken = service::StreamToken; pub type IoContext<'s, M> = service::IoContext<'s, M>; -pub type Message = service::UserMessage; pub type IoService = service::IoService; -pub type IoHandler = service::IoHandler; +pub const USER_TOKEN_START: usize = service::USER_TOKEN; diff --git a/src/io/service.rs b/src/io/service.rs index 4ecc34723..66814ba3c 100644 --- a/src/io/service.rs +++ b/src/io/service.rs @@ -4,24 +4,7 @@ use mio::util::{Slab}; use hash::*; use rlp::*; use error::*; -use io::IoError; - -/// Generic IO handler. -/// All the handler function are called from within IO event loop. -pub trait IoHandler: Send where M: Send + 'static { - /// Initialize the hadler - fn initialize(&mut self, _io: &mut IoContext) {} - /// Timer function called after a timeout created with `HandlerIo::timeout`. - fn timeout(&mut self, _io: &mut IoContext, _timer: TimerToken) {} - /// Called when a broadcasted message is received. The message can only be sent from a different IO handler. - fn message(&mut self, _io: &mut IoContext, _message: &M) {} - /// Called when an IO stream gets closed - fn stream_hup(&mut self, _io: &mut IoContext, _stream: StreamToken) {} - /// Called when an IO stream can be read from - fn stream_readable(&mut self, _io: &mut IoContext, _stream: StreamToken) {} - /// Called when an IO stream can be written to - fn stream_writable(&mut self, _io: &mut IoContext, _stream: StreamToken) {} -} +use io::{IoError, IoHandler}; pub type TimerToken = usize; pub type StreamToken = usize; @@ -30,9 +13,10 @@ pub type StreamToken = usize; const MAX_USER_TIMERS: usize = 32; const USER_TIMER: usize = 0; const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; +pub const USER_TOKEN: usize = LAST_USER_TIMER + 1; /// Messages used to communicate with the event loop from other threads. -pub enum IoMessage { +pub enum IoMessage where M: Send + Sized { /// Shutdown the event loop Shutdown, /// Register a new protocol handler. @@ -40,18 +24,13 @@ pub enum IoMessage { handler: Box+Send>, }, /// Broadcast a message across all protocol handlers. - UserMessage(UserMessage), -} - -/// User -pub struct UserMessage { - pub data: M, + UserMessage(M) } /// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. pub struct IoContext<'s, M> where M: Send + 'static { timers: &'s mut Slab, - event_loop: &'s mut EventLoop>, + pub event_loop: &'s mut EventLoop>, } impl<'s, M> IoContext<'s, M> where M: Send + 'static { @@ -78,9 +57,7 @@ impl<'s, M> IoContext<'s, M> where M: Send + 'static { /// Broadcast a message to other IO clients pub fn message(&mut self, message: M) { - match self.event_loop.channel().send(IoMessage::UserMessage(UserMessage { - data: message - })) { + match self.event_loop.channel().send(IoMessage::UserMessage(message)) { Ok(_) => {} Err(e) => { panic!("Error sending io message {:?}", e); } } @@ -116,17 +93,17 @@ impl Handler for IoManager where M: Send + 'static { fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { if events.is_hup() { for h in self.handlers.iter_mut() { - h.stream_hup(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.stream_hup(IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } else if events.is_readable() { for h in self.handlers.iter_mut() { - h.stream_readable(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.stream_readable(IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } else if events.is_writable() { for h in self.handlers.iter_mut() { - h.stream_writable(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.stream_writable(IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } } @@ -139,29 +116,30 @@ impl Handler for IoManager where M: Send + 'static { timer.delay }; for h in self.handlers.iter_mut() { - h.timeout(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.timeout(IoContext::new(event_loop, &mut self.timers), token.as_usize()); } event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); } _ => { // Just pass the event down. IoHandler is supposed to re-register it if required. for h in self.handlers.iter_mut() { - h.timeout(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.timeout(IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } } } fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { - match msg { + let mut m = msg; + match m { IoMessage::Shutdown => event_loop.shutdown(), IoMessage::AddHandler { handler, } => { self.handlers.push(handler); }, - IoMessage::UserMessage(message) => { + IoMessage::UserMessage(ref mut data) => { for h in self.handlers.iter_mut() { - h.message(&mut IoContext::new(event_loop, &mut self.timers), &message.data); + h.message(IoContext::new(event_loop, &mut self.timers), data); } } } @@ -196,6 +174,12 @@ impl IoService where M: Send + 'static { })); Ok(()) } + + /// Send a message over the network. Normaly `HostIo::send` should be used. This can be used from non-io threads. + pub fn send_message(&mut self, message: M) -> Result<(), IoError> { + try!(self.host_channel.send(IoMessage::UserMessage(message))); + Ok(()) + } } impl Drop for IoService where M: Send { diff --git a/src/network/connection.rs b/src/network/connection.rs index 2c671a08a..f11c10384 100644 --- a/src/network/connection.rs +++ b/src/network/connection.rs @@ -1,14 +1,13 @@ use std::collections::VecDeque; -use mio::{Token, EventSet, EventLoop, Timeout, PollOpt, TryRead, TryWrite}; +use mio::{Handler, Token, EventSet, EventLoop, Timeout, PollOpt, TryRead, TryWrite}; use mio::tcp::*; use hash::*; use sha3::*; use bytes::*; use rlp::*; use std::io::{self, Cursor, Read}; -use network::host::{Host}; use error::*; -use network::NetworkError; +use network::error::NetworkError; use network::handshake::Handshake; use crypto; use rcrypto::blockmodes::*; @@ -133,7 +132,7 @@ impl Connection { } /// Register this connection with the IO event loop. - pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + pub fn register(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection register; token={:?}", self.token); self.interest.insert(EventSet::readable()); event_loop.register(&self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { @@ -143,7 +142,7 @@ impl Connection { } /// Update connection registration. Should be called at the end of the IO handler. - pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> io::Result<()> { trace!(target: "net", "connection reregister; token={:?}", self.token); event_loop.reregister( &self.socket, self.token, self.interest, PollOpt::edge() | PollOpt::oneshot()).or_else(|e| { error!("Failed to reregister {:?}, {:?}", self.token, e); @@ -338,7 +337,7 @@ impl EncryptedConnection { } /// Readable IO handler. Tracker receive status and returns decoded packet if avaialable. - pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, UtilError> { + pub fn readable(&mut self, event_loop: &mut EventLoop) -> Result, UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); match self.read_state { EncryptedConnectionState::Header => { @@ -364,14 +363,14 @@ impl EncryptedConnection { } /// Writable IO handler. Processes send queeue. - pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { + pub fn writable(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); Ok(()) } /// Register this connection with the event handler. - pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { + pub fn register>(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.connection.expect(ENCRYPTED_HEADER_LEN); self.idle_timeout.map(|t| event_loop.clear_timeout(t)); self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); @@ -380,7 +379,7 @@ impl EncryptedConnection { } /// Update connection registration. This should be called at the end of the event loop. - pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { try!(self.connection.reregister(event_loop)); Ok(()) } diff --git a/src/network/discovery.rs b/src/network/discovery.rs index f2d139f45..d2a4f21a2 100644 --- a/src/network/discovery.rs +++ b/src/network/discovery.rs @@ -10,7 +10,7 @@ use mio::udp::*; use hash::*; use sha3::Hashable; use crypto::*; -use network::host::*; +use network::node::*; const ADDRESS_BYTES_SIZE: u32 = 32; ///< Size of address type in bytes. const ADDRESS_BITS: u32 = 8 * ADDRESS_BYTES_SIZE; ///< Denoted by n in [Kademlia]. @@ -70,14 +70,14 @@ impl Discovery { self.node_buckets[Discovery::distance(&self.id, &id) as usize].nodes.push(id.clone()); } - fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { + fn start_node_discovery(&mut self, event_loop: &mut EventLoop) { self.discovery_round = 0; self.discovery_id.randomize(); self.discovery_nodes.clear(); self.discover(event_loop); } - fn discover(&mut self, event_loop: &mut EventLoop) { + fn discover(&mut self, event_loop: &mut EventLoop) { if self.discovery_round == DISCOVERY_MAX_STEPS { debug!("Restarting discovery"); diff --git a/src/network/handshake.rs b/src/network/handshake.rs index af67f7ddf..ca95808b4 100644 --- a/src/network/handshake.rs +++ b/src/network/handshake.rs @@ -6,9 +6,10 @@ use bytes::Bytes; use crypto::*; use crypto; use network::connection::{Connection}; -use network::host::{NodeId, Host, HostInfo}; +use network::host::{HostInfo}; +use network::node::NodeId; use error::*; -use network::NetworkError; +use network::error::NetworkError; #[derive(PartialEq, Eq, Debug)] enum HandshakeState { @@ -88,7 +89,7 @@ impl Handshake { } /// Readable IO handler. Drives the state change. - pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), UtilError> { + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); match self.state { HandshakeState::ReadingAuth => { @@ -118,7 +119,7 @@ impl Handshake { } /// Writabe IO handler. - pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); try!(self.connection.writable()); if self.state != HandshakeState::StartSession { @@ -128,7 +129,7 @@ impl Handshake { } /// Register the IO handler with the event loop - pub fn register(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { + pub fn register>(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.idle_timeout.map(|t| event_loop.clear_timeout(t)); self.idle_timeout = event_loop.timeout_ms(self.connection.token, 1800).ok(); try!(self.connection.register(event_loop)); diff --git a/src/network/host.rs b/src/network/host.rs index 11dc491b2..d61ef614c 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -1,6 +1,7 @@ -use std::net::{SocketAddr, ToSocketAddrs}; +use std::mem; +use std::net::{SocketAddr }; use std::collections::{HashMap}; -use std::hash::{Hash, Hasher}; +use std::hash::{Hasher}; use std::str::{FromStr}; use mio::*; use mio::util::{Slab}; @@ -10,23 +11,18 @@ use hash::*; use crypto::*; use sha3::Hashable; use rlp::*; -use time::Tm; use network::handshake::Handshake; use network::session::{Session, SessionData}; use error::*; -use network::ProtocolHandler; +use io::*; +use network::NetworkProtocolHandler; +use network::node::*; const _DEFAULT_PORT: u16 = 30304; const MAX_CONNECTIONS: usize = 1024; -const MAX_USER_TIMERS: usize = 32; const IDEAL_PEERS: u32 = 10; -/// Node public key -pub type NodeId = H512; -/// IO Timer id -pub type TimerToken = usize; - #[derive(Debug)] struct NetworkConfiguration { listen_address: SocketAddr, @@ -48,88 +44,15 @@ impl NetworkConfiguration { } } -#[derive(Debug)] -/// Noe address info -pub struct NodeEndpoint { - /// IP(V4 or V6) address - address: SocketAddr, - /// Address as string (can be host name). - address_str: String, - /// Conneciton port. - udp_port: u16 -} - -impl NodeEndpoint { - /// Create endpoint from string. Performs name resolution if given a host name. - fn from_str(s: &str) -> Result { - let address = s.to_socket_addrs().map(|mut i| i.next()); - match address { - Ok(Some(a)) => Ok(NodeEndpoint { - address: a, - address_str: s.to_string(), - udp_port: a.port() - }), - Ok(_) => Err(UtilError::AddressResolve(None)), - Err(e) => Err(UtilError::AddressResolve(Some(e))) - } - } -} - -#[derive(PartialEq, Eq, Copy, Clone)] -enum PeerType { - Required, - Optional -} - -struct Node { - id: NodeId, - endpoint: NodeEndpoint, - peer_type: PeerType, - last_attempted: Option, -} - -impl FromStr for Node { - type Err = UtilError; - fn from_str(s: &str) -> Result { - let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { - (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) - } - else { - (NodeId::new(), try!(NodeEndpoint::from_str(s))) - }; - - Ok(Node { - id: id, - endpoint: endpoint, - peer_type: PeerType::Optional, - last_attempted: None, - }) - } -} - -impl PartialEq for Node { - fn eq(&self, other: &Self) -> bool { - self.id == other.id - } -} -impl Eq for Node { } - -impl Hash for Node { - fn hash(&self, state: &mut H) where H: Hasher { - self.id.hash(state) - } -} - // Tokens -const TCP_ACCEPT: usize = 1; -const IDLE: usize = 3; -const NODETABLE_RECEIVE: usize = 4; -const NODETABLE_MAINTAIN: usize = 5; -const NODETABLE_DISCOVERY: usize = 6; -const FIRST_CONNECTION: usize = 7; +const TOKEN_BEGIN: usize = USER_TOKEN_START; +const TCP_ACCEPT: usize = TOKEN_BEGIN; +const IDLE: usize = TOKEN_BEGIN + 1; +const NODETABLE_RECEIVE: usize = TOKEN_BEGIN + 2; +const NODETABLE_MAINTAIN: usize = TOKEN_BEGIN + 3; +const NODETABLE_DISCOVERY: usize = TOKEN_BEGIN + 4; +const FIRST_CONNECTION: usize = TOKEN_BEGIN + 16; const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; -const USER_TIMER: usize = LAST_CONNECTION; -const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; /// Protocol handler level packet id pub type PacketId = u8; @@ -137,12 +60,10 @@ pub type PacketId = u8; pub type ProtocolId = &'static str; /// Messages used to communitate with the event loop from other threads. -pub enum HostMessage { - /// Shutdown the event loop - Shutdown, +pub enum NetworkIoMessage where Message: Send { /// Register a new protocol handler. AddHandler { - handler: Box, + handler: Option+Send>>, protocol: ProtocolId, versions: Vec, }, @@ -153,19 +74,11 @@ pub enum HostMessage { protocol: ProtocolId, data: Vec, }, - /// Broadcast a message across the protocol handlers. - UserMessage(UserMessage), -} - -/// Id for broadcast message -pub type UserMessageId = u32; - -/// User -pub struct UserMessage { - /// ID of a protocol - pub protocol: ProtocolId, - pub id: UserMessageId, - pub data: Option>, + /// User message + User { + protocol: ProtocolId, + message: Message, + }, } /// Local (temporary) peer session ID. @@ -190,31 +103,38 @@ impl Encodable for CapabilityInfo { } /// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. -pub struct HostIo<'s> { - protocol: ProtocolId, +pub struct NetworkContext<'s, Message> where Message: Send + 'static { + io: IoContext<'s, NetworkIoMessage>, + protocol: Option, connections: &'s mut Slab, - timers: &'s mut Slab, - session: Option, - event_loop: &'s mut EventLoop, + timers: &'s mut HashMap, + session: Option, } -impl<'s> HostIo<'s> { - /// Create a new IO access point. Takes references to all the data that can be updated within the IO handler. - fn new(protocol: ProtocolId, session: Option, event_loop: &'s mut EventLoop, connections: &'s mut Slab, timers: &'s mut Slab) -> HostIo<'s> { - HostIo { +impl<'s, Message> NetworkContext<'s, Message> where Message: Send + 'static, { + /// Create a new network IO access point. Takes references to all the data that can be updated within the IO handler. + fn new(io: IoContext<'s, NetworkIoMessage>, + protocol: Option, + session: Option, connections: &'s mut Slab, + timers: &'s mut HashMap) -> NetworkContext<'s, Message> { + NetworkContext { + io: io, protocol: protocol, session: session, - event_loop: event_loop, connections: connections, timers: timers, } } + fn set_protocol(&mut self, protocol: ProtocolId) { + self.protocol = Some(protocol); + } + /// Send a packet over the network to another peer. pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { match self.connections.get_mut(Token(peer)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { + s.send_packet(self.protocol.unwrap(), packet_id as u8, &data).unwrap_or_else(|e| { warn!(target: "net", "Send error: {:?}", e); }); //TODO: don't copy vector data }, @@ -228,49 +148,28 @@ impl<'s> HostIo<'s> { /// Respond to a current network message. Panics if no there is no packet in the context. pub fn respond(&mut self, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { match self.session { - Some(session) => self.send(session.as_usize(), packet_id, data), + Some(session) => self.send(session, packet_id, data), None => { panic!("Respond: Session does not exist") } } } - /// Register a new IO timer. Returns a new timer toke. 'ProtocolHandler::timeout' will be called with the token. - pub fn register_timer(&mut self, ms: u64) -> Result{ - match self.timers.insert(UserTimer { - delay: ms, - protocol: self.protocol, - }) { - Ok(token) => { - self.event_loop.timeout_ms(token, ms).expect("Error registering user timer"); - Ok(token.as_usize()) - }, - _ => { panic!("Max timers reached") } - } - } - - /// Broadcast a message to other IO clients - pub fn message(&mut self, id: UserMessageId, data: Option>) { - match self.event_loop.channel().send(HostMessage::UserMessage(UserMessage { - protocol: self.protocol, - id: id, - data: data - })) { - Ok(_) => {} - Err(e) => { panic!("Error sending io message {:?}", e); } - } - } - /// Disable current protocol capability for given peer. If no capabilities left peer gets disconnected. pub fn disable_peer(&mut self, _peer: PeerId) { //TODO: remove capability, disconnect if no capabilities left } -} - -struct UserTimer { - protocol: ProtocolId, - delay: u64, + /// Register a new IO timer. Returns a new timer token. 'NetworkProtocolHandler::timeout' will be called with the token. + pub fn register_timer(&mut self, ms: u64) -> Result{ + match self.io.register_timer(ms) { + Ok(token) => { + self.timers.insert(token, self.protocol.unwrap()); + Ok(token) + }, + e @ Err(_) => e, + } + } } /// Shared host information @@ -315,67 +214,40 @@ enum ConnectionEntry { } /// Root IO handler. Manages protocol handlers, IO timers and network connections. -pub struct Host { +pub struct Host where Message: Send { info: HostInfo, - _udp_socket: UdpSocket, - _listener: TcpListener, + udp_socket: UdpSocket, + listener: TcpListener, connections: Slab, - timers: Slab, + timers: HashMap, nodes: HashMap, - handlers: HashMap>, - _idle_timeout: Timeout, + handlers: HashMap>>, } -impl Host { - /// Creates a new instance and registers it with the event loop. - pub fn start(event_loop: &mut EventLoop) -> Result<(), UtilError> { +impl Host where Message: Send { + pub fn new() -> Host { let config = NetworkConfiguration::new(); - /* - match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { - Some(iface) => config.public_address = iface.addr.unwrap(), - None => warn!("No public network interface"), - */ - let addr = config.listen_address; // Setup the server socket let listener = TcpListener::bind(&addr).unwrap(); - // Start listening for incoming connections - event_loop.register(&listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); - let idle_timeout = event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay - // open the udp socket let udp_socket = UdpSocket::bound(&addr).unwrap(); - event_loop.register(&udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); - event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); - let port = config.listen_address.port(); - - let mut host = Host { + Host:: { info: HostInfo { keys: KeyPair::create().unwrap(), config: config, nonce: H256::random(), protocol_version: 4, client_version: "parity".to_string(), - listen_port: port, + listen_port: 0, capabilities: Vec::new(), }, - _udp_socket: udp_socket, - _listener: listener, + udp_socket: udp_socket, + listener: listener, connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), - timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), + timers: HashMap::new(), nodes: HashMap::new(), handlers: HashMap::new(), - _idle_timeout: idle_timeout, - }; - - host.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); - host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); - host.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); - host.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); - host.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); - host.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); - - try!(event_loop.run(&mut host)); - Ok(()) + } } fn add_node(&mut self, id: &str) { @@ -387,8 +259,8 @@ impl Host { } } - fn maintain_network(&mut self, event_loop: &mut EventLoop) { - self.connect_peers(event_loop); + fn maintain_network(&mut self, io: IoContext>) { + self.connect_peers(io); } fn have_session(&self, id: &NodeId) -> bool { @@ -399,8 +271,7 @@ impl Host { self.connections.iter().any(|e| match e { &ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) } - fn connect_peers(&mut self, event_loop: &mut EventLoop) { - + fn connect_peers(&mut self, mut io: IoContext>) { struct NodeInfo { id: NodeId, peer_type: PeerType @@ -425,7 +296,7 @@ impl Host { for n in to_connect.iter() { if n.peer_type == PeerType::Required { if req_conn < IDEAL_PEERS { - self.connect_peer(&n.id, event_loop); + self.connect_peer(&n.id, &mut io); } req_conn += 1; } @@ -440,14 +311,14 @@ impl Host { for n in to_connect.iter() { if n.peer_type == PeerType::Optional && open_slots > 0 { open_slots -= 1; - self.connect_peer(&n.id, event_loop); + self.connect_peer(&n.id, &mut io); } } } } } - fn connect_peer(&mut self, id: &NodeId, event_loop: &mut EventLoop) { + fn connect_peer(&mut self, id: &NodeId, io: &mut IoContext>) { if self.have_session(id) { warn!("Aborted connect. Node already connected."); @@ -478,7 +349,7 @@ impl Host { match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.start(&self.info, true) - .and_then(|_| h.register(event_loop)) + .and_then(|_| h.register(io.event_loop)) .unwrap_or_else (|e| { debug!(target: "net", "Handshake create error: {:?}", e); }); @@ -491,23 +362,23 @@ impl Host { } - fn accept(&mut self, _event_loop: &mut EventLoop) { + fn accept(&mut self, _io: IoContext>) { trace!(target: "net", "accept"); } - fn connection_writable(&mut self, token: Token, event_loop: &mut EventLoop) { + fn connection_writable<'s>(&'s mut self, token: StreamToken, io: IoContext<'s, NetworkIoMessage>) { let mut kill = false; let mut create_session = false; - match self.connections.get_mut(token) { + match self.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.writable(event_loop, &self.info).unwrap_or_else(|e| { + h.writable(io.event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Handshake write error: {:?}", e); kill = true; }); create_session = h.done(); }, Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.writable(event_loop, &self.info).unwrap_or_else(|e| { + s.writable(io.event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Session write error: {:?}", e); kill = true; }); @@ -516,39 +387,41 @@ impl Host { warn!(target: "net", "Received event for unknown connection"); } } + let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); if kill { - self.kill_connection(token, event_loop); + Host::kill_connection(token, &mut net_context, &mut self.handlers); } if create_session { - self.start_session(token, event_loop); + Host::start_session(&self.info, token, &mut net_context); } - match self.connections.get_mut(token) { + match net_context.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.reregister(event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + s.reregister(net_context.io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); }, _ => (), } } - fn connection_closed(&mut self, token: Token, event_loop: &mut EventLoop) { - self.kill_connection(token, event_loop); + fn connection_closed<'s>(&'s mut self, token: TimerToken, io: IoContext<'s, NetworkIoMessage>) { + let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); + Host::kill_connection(token, &mut net_context, &mut self.handlers); } - fn connection_readable(&mut self, token: Token, event_loop: &mut EventLoop) { + fn connection_readable<'s>(&'s mut self, token: StreamToken, io: IoContext<'s, NetworkIoMessage>) { let mut kill = false; let mut create_session = false; let mut ready_data: Vec = Vec::new(); let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; - match self.connections.get_mut(token) { + match self.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { - h.readable(event_loop, &self.info).unwrap_or_else(|e| { + h.readable(io.event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Handshake read error: {:?}", e); kill = true; }); create_session = h.done(); }, Some(&mut ConnectionEntry::Session(ref mut s)) => { - let sd = { s.readable(event_loop, &self.info).unwrap_or_else(|e| { + let sd = { s.readable(io.event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Session read error: {:?}", e); kill = true; SessionData::None @@ -578,32 +451,35 @@ impl Host { warn!(target: "net", "Received event for unknown connection"); } } + let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); if kill { - self.kill_connection(token, event_loop); + Host::kill_connection(token, &mut net_context, &mut self.handlers); } if create_session { - self.start_session(token, event_loop); + Host::start_session(&self.info, token, &mut net_context); } for p in ready_data { let mut h = self.handlers.get_mut(p).unwrap(); - h.connected(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize()); + net_context.set_protocol(p); + h.connected(&mut net_context, &token); } if let Some((p, packet_id, data)) = packet_data { let mut h = self.handlers.get_mut(p).unwrap(); - h.read(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize(), packet_id, &data[1..]); + net_context.set_protocol(p); + h.read(&mut net_context, &token, packet_id, &data[1..]); } - match self.connections.get_mut(token) { + match net_context.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.reregister(event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + s.reregister(net_context.io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); }, _ => (), } } - fn start_session(&mut self, token: Token, event_loop: &mut EventLoop) { - let info = &self.info; - self.connections.replace_with(token, |c| { + fn start_session(info: &HostInfo, token: StreamToken, io: &mut NetworkContext) { + let event_loop = &mut io.io.event_loop; + io.connections.replace_with(Token(token), |c| { match c { ConnectionEntry::Handshake(h) => Session::new(h, event_loop, info) .map(|s| Some(ConnectionEntry::Session(s))) @@ -616,16 +492,17 @@ impl Host { }).expect("Error updating slab with session"); } - fn connection_timeout(&mut self, token: Token, event_loop: &mut EventLoop) { - self.kill_connection(token, event_loop) + fn connection_timeout<'s>(&'s mut self, token: StreamToken, io: IoContext<'s, NetworkIoMessage>) { + let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); + Host::kill_connection(token, &mut net_context, &mut self.handlers) } - fn kill_connection(&mut self, token: Token, event_loop: &mut EventLoop) { + fn kill_connection(token: StreamToken, io: &mut NetworkContext, handlers: &mut HashMap>>) { let mut to_disconnect: Vec = Vec::new(); - match self.connections.get_mut(token) { + match io.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Handshake(_)) => (), // just abandon handshake Some(&mut ConnectionEntry::Session(ref mut s)) if s.is_ready() => { - for (p, _) in self.handlers.iter_mut() { + for (p, _) in handlers.iter_mut() { if s.have_capability(p) { to_disconnect.push(p); } @@ -634,87 +511,103 @@ impl Host { _ => (), } for p in to_disconnect { - let mut h = self.handlers.get_mut(p).unwrap(); - h.disconnected(&mut HostIo::new(p, Some(token), event_loop, &mut self.connections, &mut self.timers), &token.as_usize()); + let mut h = handlers.get_mut(p).unwrap(); + io.set_protocol(p); + h.disconnected(io, &token); } - self.connections.remove(token); + io.connections.remove(Token(token)); } } -impl Handler for Host { - type Timeout = Token; - type Message = HostMessage; +impl IoHandler> for Host where Message: Send + 'static { + /// Initialize networking + fn initialize(&mut self, io: IoContext>) { + /* + match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { + Some(iface) => config.public_address = iface.addr.unwrap(), + None => warn!("No public network interface"), + */ - fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { - if events.is_hup() { - trace!(target: "net", "hup"); - match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_closed(token, event_loop), - _ => warn!(target: "net", "Unexpected hup"), - }; - } - else if events.is_readable() { - match token.as_usize() { - TCP_ACCEPT => self.accept(event_loop), - IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(token, event_loop), - NODETABLE_RECEIVE => {}, - _ => panic!("Received unknown readable token"), - } - } - else if events.is_writable() { - match token.as_usize() { - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(token, event_loop), - _ => panic!("Received unknown writable token"), - } + // Start listening for incoming connections + io.event_loop.register(&self.listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); + io.event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay + // open the udp socket + io.event_loop.register(&self.udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); + io.event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); + let port = self.info.config.listen_address.port(); + self.info.listen_port = port; + + self.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); + self.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); + self.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); + self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); + self.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); + self.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); + } + + fn stream_hup(&mut self, io: IoContext>, stream: StreamToken) { + trace!(target: "net", "Hup: {}", stream); + match stream { + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_closed(stream, io), + _ => warn!(target: "net", "Unexpected hup"), + }; + } + + fn stream_readable(&mut self, io: IoContext>, stream: StreamToken) { + match stream { + TCP_ACCEPT => self.accept(io), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(stream, io), + NODETABLE_RECEIVE => {}, + _ => panic!("Received unknown readable token"), } } - fn timeout(&mut self, event_loop: &mut EventLoop, token: Token) { - match token.as_usize() { - IDLE => self.maintain_network(event_loop), - FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, event_loop), + fn stream_writable(&mut self, io: IoContext>, stream: StreamToken) { + match stream { + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(stream, io), + _ => panic!("Received unknown writable token"), + } + } + + fn timeout(&mut self, io: IoContext>, token: TimerToken) { + match token { + IDLE => self.maintain_network(io), + FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, io), NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, - USER_TIMER ... LAST_USER_TIMER => { - let (protocol, delay) = { - let timer = self.timers.get_mut(token).expect("Unknown user timer token"); - (timer.protocol, timer.delay) - }; + _ => { + let protocol: ProtocolId = self.timers.get_mut(&token).expect("Unknown user timer token"); match self.handlers.get_mut(protocol) { None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, Some(h) => { - h.timeout(&mut HostIo::new(protocol, None, event_loop, &mut self.connections, &mut self.timers), token.as_usize()); - event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); + h.timeout(&mut NetworkContext::new(io, Some(protocol), Some(token), &mut self.connections, &mut self.timers), token); } } } - _ => panic!("Unknown timer token"), } } - fn notify(&mut self, event_loop: &mut EventLoop, msg: Self::Message) { - match msg { - HostMessage::Shutdown => event_loop.shutdown(), - HostMessage::AddHandler { - handler, - protocol, - versions + fn message(&mut self, io: IoContext>, message: &mut NetworkIoMessage) { + match message { + &mut NetworkIoMessage::AddHandler { + ref mut handler, + ref protocol, + ref versions } => { - self.handlers.insert(protocol, handler); + self.handlers.insert(protocol, mem::replace(handler, None).unwrap()); for v in versions { - self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: v, packet_count:0 }); + self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: *v, packet_count:0 }); } }, - HostMessage::Send { - peer, - packet_id, - protocol, - data, + &mut NetworkIoMessage::Send { + ref peer, + ref packet_id, + ref protocol, + ref data, } => { - match self.connections.get_mut(Token(peer as usize)) { + match self.connections.get_mut(Token(*peer as usize)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(protocol, packet_id as u8, &data).unwrap_or_else(|e| { + s.send_packet(protocol, *packet_id as u8, &data).unwrap_or_else(|e| { warn!(target: "net", "Send error: {:?}", e); }); //TODO: don't copy vector data }, @@ -723,10 +616,15 @@ impl Handler for Host { } } }, - HostMessage::UserMessage(message) => { + &mut NetworkIoMessage::User { + ref protocol, + ref message + } => { + let mut net_context = NetworkContext::new(io, None, None, &mut self.connections, &mut self.timers); for (p, h) in self.handlers.iter_mut() { - if p != &message.protocol { - h.message(&mut HostIo::new(message.protocol, None, event_loop, &mut self.connections, &mut self.timers), &message); + if p != protocol { + net_context.set_protocol(p); + h.message(&mut net_context, &message); } } } diff --git a/src/network/mod.rs b/src/network/mod.rs index cdce08d00..1a1450cb6 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -9,27 +9,27 @@ /// struct MyHandler; /// /// impl ProtocolHandler for MyHandler { -/// fn initialize(&mut self, io: &mut HandlerIo) { +/// fn initialize(&mut self, io: &mut NetworkContext) { /// io.register_timer(1000); /// } /// -/// fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { +/// fn read(&mut self, io: &mut NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { /// println!("Received {} ({} bytes) from {}", packet_id, data.len(), peer); /// } /// -/// fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { +/// fn connected(&mut self, io: &mut NetworkContext, peer: &PeerId) { /// println!("Connected {}", peer); /// } /// -/// fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { +/// fn disconnected(&mut self, io: &mut NetworkContext, peer: &PeerId) { /// println!("Disconnected {}", peer); /// } /// -/// fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { +/// fn timeout(&mut self, io: &mut NetworkContext, timer: TimerToken) { /// println!("Timeout {}", timer); /// } /// -/// fn message(&mut self, io: &mut HandlerIo, message: &Message) { +/// fn message(&mut self, io: &mut NetworkContext, message: &Message) { /// println!("Message {}:{}", message.protocol, message.id); /// } /// } @@ -50,69 +50,31 @@ mod handshake; mod session; mod discovery; mod service; - -#[derive(Debug, Copy, Clone)] -pub enum DisconnectReason -{ - DisconnectRequested, - TCPError, - BadProtocol, - UselessPeer, - TooManyPeers, - DuplicatePeer, - IncompatibleProtocol, - NullIdentity, - ClientQuit, - UnexpectedIdentity, - LocalIdentity, - PingTimeout, -} - -#[derive(Debug)] -pub enum NetworkError { - Auth, - BadProtocol, - PeerNotFound, - Disconnect(DisconnectReason), - Mio(::std::io::Error), -} - -impl From<::rlp::DecoderError> for NetworkError { - fn from(_err: ::rlp::DecoderError) -> NetworkError { - NetworkError::Auth - } -} - -impl From<::mio::NotifyError> for NetworkError { - fn from(_err: ::mio::NotifyError) -> NetworkError { - NetworkError::Mio(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) - } -} +mod error; +mod node; pub type PeerId = host::PeerId; pub type PacketId = host::PacketId; -pub type TimerToken = host::TimerToken; -pub type HandlerIo<'s> = host::HostIo<'s>; -pub type Message = host::UserMessage; -pub type MessageId = host::UserMessageId; +pub type NetworkContext<'s, Message> = host::NetworkContext<'s, Message>; +pub type NetworkService = service::NetworkService; +pub type NetworkIoMessage = host::NetworkIoMessage; +pub type NetworkError = error::NetworkError; + +use io::*; /// Network IO protocol handler. This needs to be implemented for each new subprotocol. -/// TODO: Separate p2p networking IO from IPC IO. `timeout` and `message` should go to a more genera IO provider. /// All the handler function are called from within IO event loop. -pub trait ProtocolHandler: Send { - /// Initialize the hadler - fn initialize(&mut self, io: &mut HandlerIo); +/// `Message` is the type for message data. +pub trait NetworkProtocolHandler: Send where Message: Send { /// Called when new network packet received. - fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]); + fn read(&mut self, io: &mut NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]); /// Called when new peer is connected. Only called when peer supports the same protocol. - fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId); + fn connected(&mut self, io: &mut NetworkContext, peer: &PeerId); /// Called when a previously connected peer disconnects. - fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId); - /// Timer function called after a timeout created with `HandlerIo::timeout`. - fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken); - /// Called when a broadcasted message is received. The message can only be sent from a different protocol handler. - fn message(&mut self, io: &mut HandlerIo, message: &Message); + fn disconnected(&mut self, io: &mut NetworkContext, peer: &PeerId); + /// Timer function called after a timeout created with `NetworkContext::timeout`. + fn timeout(&mut self, _io: &mut NetworkContext, _timer: TimerToken) {} + /// Called when a broadcasted message is received. The message can only be sent from a different IO handler. + fn message(&mut self, _io: &mut NetworkContext, _message: &Message) {} } -pub type NetworkService = service::NetworkService; - diff --git a/src/network/service.rs b/src/network/service.rs index 23cbb57d9..be70e2ee1 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -1,32 +1,28 @@ -use std::thread::{self, JoinHandle}; -use mio::*; use error::*; -use network::{NetworkError, ProtocolHandler}; -use network::host::{Host, HostMessage, PeerId, PacketId, ProtocolId}; +use network::{NetworkProtocolHandler}; +use network::error::{NetworkError}; +use network::host::{Host, NetworkIoMessage, PeerId, PacketId, ProtocolId}; +use io::*; /// IO Service with networking -pub struct NetworkService { - thread: Option>, - host_channel: Sender +/// `Message` defines a notification data type. +pub struct NetworkService where Message: Send + 'static { + io_service: IoService>, } -impl NetworkService { +impl NetworkService where Message: Send + 'static { /// Starts IO event loop - pub fn start() -> Result { - let mut event_loop = EventLoop::new().unwrap(); - let channel = event_loop.channel(); - let thread = thread::spawn(move || { - Host::start(&mut event_loop).unwrap(); //TODO: - }); + pub fn start() -> Result, UtilError> { + let mut io_service = try!(IoService::>::start()); + try!(io_service.register_handler(Box::new(Host::new()))); Ok(NetworkService { - thread: Some(thread), - host_channel: channel + io_service: io_service }) } /// Send a message over the network. Normaly `HostIo::send` should be used. This can be used from non-io threads. pub fn send(&mut self, peer: &PeerId, packet_id: PacketId, protocol: ProtocolId, data: &[u8]) -> Result<(), NetworkError> { - try!(self.host_channel.send(HostMessage::Send { + try!(self.io_service.send_message(NetworkIoMessage::Send { peer: *peer, packet_id: packet_id, protocol: protocol, @@ -36,9 +32,9 @@ impl NetworkService { } /// Regiter a new protocol handler with the event loop. - pub fn register_protocol(&mut self, handler: Box, protocol: ProtocolId, versions: &[u8]) -> Result<(), NetworkError> { - try!(self.host_channel.send(HostMessage::AddHandler { - handler: handler, + pub fn register_protocol(&mut self, handler: Box+Send>, protocol: ProtocolId, versions: &[u8]) -> Result<(), NetworkError> { + try!(self.io_service.send_message(NetworkIoMessage::AddHandler { + handler: Some(handler), protocol: protocol, versions: versions.to_vec(), })); @@ -46,10 +42,3 @@ impl NetworkService { } } -impl Drop for NetworkService { - fn drop(&mut self) { - self.host_channel.send(HostMessage::Shutdown).unwrap(); - self.thread.take().unwrap().join().unwrap(); - } -} - diff --git a/src/network/session.rs b/src/network/session.rs index 96390d366..2d8ba2245 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -4,8 +4,9 @@ use rlp::*; use network::connection::{EncryptedConnection, Packet}; use network::handshake::Handshake; use error::*; -use network::{NetworkError, DisconnectReason}; +use network::error::{NetworkError, DisconnectReason}; use network::host::*; +use network::node::NodeId; /// Peer session over encrypted connection. /// When created waits for Hello packet exchange and signals ready state. @@ -83,7 +84,7 @@ const PACKET_LAST: u8 = 0x7f; impl Session { /// Create a new session out of comepleted handshake. Consumes handshake object. - pub fn new(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { + pub fn new>(h: Handshake, event_loop: &mut EventLoop, host: &HostInfo) -> Result { let id = h.id.clone(); let connection = try!(EncryptedConnection::new(h)); let mut session = Session { @@ -108,7 +109,7 @@ impl Session { } /// Readable IO handler. Returns packet data if available. - pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { + pub fn readable(&mut self, event_loop: &mut EventLoop, host: &HostInfo) -> Result { match try!(self.connection.readable(event_loop)) { Some(data) => Ok(try!(self.read_packet(data, host))), None => Ok(SessionData::None) @@ -116,7 +117,7 @@ impl Session { } /// Writable IO handler. Sends pending packets. - pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { + pub fn writable(&mut self, event_loop: &mut EventLoop, _host: &HostInfo) -> Result<(), UtilError> { self.connection.writable(event_loop) } @@ -126,7 +127,7 @@ impl Session { } /// Update registration with the event loop. Should be called at the end of the IO handler. - pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { + pub fn reregister(&mut self, event_loop: &mut EventLoop) -> Result<(), UtilError> { self.connection.reregister(event_loop) } From 4be539a9656c35ad6d928333eab3c8dd62522454 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 12:16:10 +0100 Subject: [PATCH 326/379] Stirct ordering for hashes. --- src/hash.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 087454ceb..5eb76a05e 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -224,16 +224,22 @@ macro_rules! impl_hash { } } - impl PartialOrd for $from { - fn partial_cmp(&self, other: &Self) -> Option { + impl Ord for $from { + fn cmp(&self, other: &Self) -> Ordering { for i in 0..$size { if self.0[i] > other.0[i] { - return Some(Ordering::Greater); + return Ordering::Greater; } else if self.0[i] < other.0[i] { - return Some(Ordering::Less); + return Ordering::Less; } } - Some(Ordering::Equal) + Ordering::Equal + } + } + + impl PartialOrd for $from { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } } From d5ed4a8ffd941371e8c4eb0273cc80ccad7bb26c Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Wed, 13 Jan 2016 12:24:14 +0100 Subject: [PATCH 327/379] Fixed typos --- src/hash.rs | 2 +- src/uint.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 087454ceb..ad2d19956 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -381,7 +381,7 @@ impl<'a> From<&'a U256> for H256 { value.to_bytes(&mut ret); ret } - } + } } impl From for Address { diff --git a/src/uint.rs b/src/uint.rs index b7411fd30..8240a20a3 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -71,7 +71,7 @@ macro_rules! construct_uint { pub fn as_u32(&self) -> u32 { let &$name(ref arr) = self; if (arr[0] & (0xffffffffu64 << 32)) != 0 { - panic!("Intger overflow when casting U256") + panic!("Integer overflow when casting U256") } self.as_u64() as u32 } @@ -82,7 +82,7 @@ macro_rules! construct_uint { let &$name(ref arr) = self; for i in 1..$n_words { if arr[i] != 0 { - panic!("Intger overflow when casting U256") + panic!("Integer overflow when casting U256") } } arr[0] From 28c482691a6e2b95c80d120d82c1fc693f30b427 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 13 Jan 2016 13:56:48 +0100 Subject: [PATCH 328/379] Fixed context lifetimes --- src/io/mod.rs | 87 +++++++++++++++++++------------ src/io/service.rs | 58 ++++++++++----------- src/lib.rs | 1 + src/network/host.rs | 122 ++++++++++++++++++++------------------------ src/network/mod.rs | 29 ++++++----- 5 files changed, 158 insertions(+), 139 deletions(-) diff --git a/src/io/mod.rs b/src/io/mod.rs index 72906b6f4..497e1ae78 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -3,47 +3,38 @@ /// Example usage for craeting a network service and adding an IO handler: /// /// ```rust -/// extern crate ethcore_util as util; -/// use util::network::*; +/// extern crate ethcore_util; +/// use ethcore_util::*; /// /// struct MyHandler; /// -/// impl ProtocolHandler for MyHandler { -/// fn initialize(&mut self, io: &mut HandlerIo) { -/// io.register_timer(1000); +/// struct MyMessage { +/// data: u32 +/// } +/// +/// impl IoHandler for MyHandler { +/// fn initialize(&mut self, io: &mut IoContext) { +/// io.register_timer(1000).unwrap(); /// } /// -/// fn read(&mut self, io: &mut HandlerIo, peer: &PeerId, packet_id: u8, data: &[u8]) { -/// println!("Received {} ({} bytes) from {}", packet_id, data.len(), peer); -/// } -/// -/// fn connected(&mut self, io: &mut HandlerIo, peer: &PeerId) { -/// println!("Connected {}", peer); -/// } -/// -/// fn disconnected(&mut self, io: &mut HandlerIo, peer: &PeerId) { -/// println!("Disconnected {}", peer); -/// } -/// -/// fn timeout(&mut self, io: &mut HandlerIo, timer: TimerToken) { +/// fn timeout(&mut self, _io: &mut IoContext, timer: TimerToken) { /// println!("Timeout {}", timer); /// } /// -/// fn message(&mut self, io: &mut HandlerIo, message: &Message) { -/// println!("Message {}:{}", message.protocol, message.id); +/// fn message(&mut self, _io: &mut IoContext, message: &mut MyMessage) { +/// println!("Message {}", message.data); /// } -/// } +/// } /// /// fn main () { -/// let mut service = NetworkService::start().expect("Error creating network service"); -/// service.register_protocol(Box::new(MyHandler), "myproto", &[1u8]); +/// let mut service = IoService::::start().expect("Error creating network service"); +/// service.register_handler(Box::new(MyHandler)).unwrap(); /// /// // Wait for quit condition /// // ... /// // Drop the service /// } /// ``` -extern crate mio; mod service; #[derive(Debug)] @@ -51,8 +42,8 @@ pub enum IoError { Mio(::std::io::Error), } -impl From<::mio::NotifyError>> for IoError where M: Send { - fn from(_err: ::mio::NotifyError>) -> IoError { +impl From<::mio::NotifyError>> for IoError where Message: Send { + fn from(_err: ::mio::NotifyError>) -> IoError { IoError::Mio(::std::io::Error::new(::std::io::ErrorKind::ConnectionAborted, "Network IO notification error")) } } @@ -62,17 +53,17 @@ impl From<::mio::NotifyError>> for IoError where M: Sen /// `Message` type is used as notification data pub trait IoHandler: Send where Message: Send + 'static { /// Initialize the handler - fn initialize(&mut self, _io: IoContext) {} + fn initialize<'s>(&'s mut self, _io: &mut IoContext<'s, Message>) {} /// Timer function called after a timeout created with `HandlerIo::timeout`. - fn timeout(&mut self, _io: IoContext, _timer: TimerToken) {} + fn timeout<'s>(&'s mut self, _io: &mut IoContext<'s, Message>, _timer: TimerToken) {} /// Called when a broadcasted message is received. The message can only be sent from a different IO handler. - fn message(&mut self, _io: IoContext, _message: &mut Message) {} // TODO: make message immutable and provide internal channel for adding network handler + fn message<'s>(&'s mut self, _io: &mut IoContext<'s, Message>, _message: &'s mut Message) {} // TODO: make message immutable and provide internal channel for adding network handler /// Called when an IO stream gets closed - fn stream_hup(&mut self, _io: IoContext, _stream: StreamToken) {} + fn stream_hup<'s>(&'s mut self, _io: &mut IoContext<'s, Message>, _stream: StreamToken) {} /// Called when an IO stream can be read from - fn stream_readable(&mut self, _io: IoContext, _stream: StreamToken) {} + fn stream_readable<'s>(&'s mut self, _io: &mut IoContext<'s, Message>, _stream: StreamToken) {} /// Called when an IO stream can be written to - fn stream_writable(&mut self, _io: IoContext, _stream: StreamToken) {} + fn stream_writable<'s>(&'s mut self, _io: &mut IoContext<'s, Message>, _stream: StreamToken) {} } pub type TimerToken = service::TimerToken; @@ -83,3 +74,35 @@ pub const USER_TOKEN_START: usize = service::USER_TOKEN; +#[cfg(test)] +mod tests { + + use io::*; + + struct MyHandler; + + struct MyMessage { + data: u32 + } + + impl IoHandler for MyHandler { + fn initialize(&mut self, io: &mut IoContext) { + io.register_timer(1000).unwrap(); + } + + fn timeout(&mut self, _io: &mut IoContext, timer: TimerToken) { + println!("Timeout {}", timer); + } + + fn message(&mut self, _io: &mut IoContext, message: &mut MyMessage) { + println!("Message {}", message.data); + } + } + + #[test] + fn test_service_register_handler () { + let mut service = IoService::::start().expect("Error creating network service"); + service.register_handler(Box::new(MyHandler)).unwrap(); + } + +} diff --git a/src/io/service.rs b/src/io/service.rs index 66814ba3c..aab8f5c2d 100644 --- a/src/io/service.rs +++ b/src/io/service.rs @@ -16,26 +16,26 @@ const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; pub const USER_TOKEN: usize = LAST_USER_TIMER + 1; /// Messages used to communicate with the event loop from other threads. -pub enum IoMessage where M: Send + Sized { +pub enum IoMessage where Message: Send + Sized { /// Shutdown the event loop Shutdown, /// Register a new protocol handler. AddHandler { - handler: Box+Send>, + handler: Box+Send>, }, /// Broadcast a message across all protocol handlers. - UserMessage(M) + UserMessage(Message) } /// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. -pub struct IoContext<'s, M> where M: Send + 'static { +pub struct IoContext<'s, Message> where Message: Send + 'static { timers: &'s mut Slab, - pub event_loop: &'s mut EventLoop>, + pub event_loop: &'s mut EventLoop>, } -impl<'s, M> IoContext<'s, M> where M: Send + 'static { +impl<'s, Message> IoContext<'s, Message> where Message: Send + 'static { /// Create a new IO access point. Takes references to all the data that can be updated within the IO handler. - fn new(event_loop: &'s mut EventLoop>, timers: &'s mut Slab) -> IoContext<'s, M> { + fn new(event_loop: &'s mut EventLoop>, timers: &'s mut Slab) -> IoContext<'s, Message> { IoContext { event_loop: event_loop, timers: timers, @@ -56,7 +56,7 @@ impl<'s, M> IoContext<'s, M> where M: Send + 'static { } /// Broadcast a message to other IO clients - pub fn message(&mut self, message: M) { + pub fn message(&mut self, message: Message) { match self.event_loop.channel().send(IoMessage::UserMessage(message)) { Ok(_) => {} Err(e) => { panic!("Error sending io message {:?}", e); } @@ -69,14 +69,14 @@ struct UserTimer { } /// Root IO handler. Manages user handlers, messages and IO timers. -pub struct IoManager where M: Send { +pub struct IoManager where Message: Send { timers: Slab, - handlers: Vec>>, + handlers: Vec>>, } -impl IoManager where M: Send + 'static { +impl IoManager where Message: Send + 'static { /// Creates a new instance and registers it with the event loop. - pub fn start(event_loop: &mut EventLoop>) -> Result<(), UtilError> { + pub fn start(event_loop: &mut EventLoop>) -> Result<(), UtilError> { let mut io = IoManager { timers: Slab::new_starting_at(Token(USER_TIMER), MAX_USER_TIMERS), handlers: Vec::new(), @@ -86,24 +86,24 @@ impl IoManager where M: Send + 'static { } } -impl Handler for IoManager where M: Send + 'static { +impl Handler for IoManager where Message: Send + 'static { type Timeout = Token; - type Message = IoMessage; + type Message = IoMessage; fn ready(&mut self, event_loop: &mut EventLoop, token: Token, events: EventSet) { if events.is_hup() { for h in self.handlers.iter_mut() { - h.stream_hup(IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.stream_hup(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } else if events.is_readable() { for h in self.handlers.iter_mut() { - h.stream_readable(IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.stream_readable(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } else if events.is_writable() { for h in self.handlers.iter_mut() { - h.stream_writable(IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.stream_writable(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } } @@ -116,13 +116,13 @@ impl Handler for IoManager where M: Send + 'static { timer.delay }; for h in self.handlers.iter_mut() { - h.timeout(IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.timeout(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); } event_loop.timeout_ms(token, delay).expect("Error re-registering user timer"); } _ => { // Just pass the event down. IoHandler is supposed to re-register it if required. for h in self.handlers.iter_mut() { - h.timeout(IoContext::new(event_loop, &mut self.timers), token.as_usize()); + h.timeout(&mut IoContext::new(event_loop, &mut self.timers), token.as_usize()); } } } @@ -139,7 +139,7 @@ impl Handler for IoManager where M: Send + 'static { }, IoMessage::UserMessage(ref mut data) => { for h in self.handlers.iter_mut() { - h.message(IoContext::new(event_loop, &mut self.timers), data); + h.message(&mut IoContext::new(event_loop, &mut self.timers), data); } } } @@ -147,19 +147,19 @@ impl Handler for IoManager where M: Send + 'static { } /// General IO Service. Starts an event loop and dispatches IO requests. -/// 'M' is a notification message type -pub struct IoService where M: Send + 'static { +/// 'Message' is a notification message type +pub struct IoService where Message: Send + 'static { thread: Option>, - host_channel: Sender> + host_channel: Sender> } -impl IoService where M: Send + 'static { +impl IoService where Message: Send + 'static { /// Starts IO event loop - pub fn start() -> Result, UtilError> { + pub fn start() -> Result, UtilError> { let mut event_loop = EventLoop::new().unwrap(); let channel = event_loop.channel(); let thread = thread::spawn(move || { - IoManager::::start(&mut event_loop).unwrap(); //TODO: + IoManager::::start(&mut event_loop).unwrap(); //TODO: }); Ok(IoService { thread: Some(thread), @@ -168,7 +168,7 @@ impl IoService where M: Send + 'static { } /// Regiter a IO hadnler with the event loop. - pub fn register_handler(&mut self, handler: Box+Send>) -> Result<(), IoError> { + pub fn register_handler(&mut self, handler: Box+Send>) -> Result<(), IoError> { try!(self.host_channel.send(IoMessage::AddHandler { handler: handler, })); @@ -176,13 +176,13 @@ impl IoService where M: Send + 'static { } /// Send a message over the network. Normaly `HostIo::send` should be used. This can be used from non-io threads. - pub fn send_message(&mut self, message: M) -> Result<(), IoError> { + pub fn send_message(&mut self, message: Message) -> Result<(), IoError> { try!(self.host_channel.send(IoMessage::UserMessage(message))); Ok(()) } } -impl Drop for IoService where M: Send { +impl Drop for IoService where Message: Send { fn drop(&mut self) { self.host_channel.send(IoMessage::Shutdown).unwrap(); self.thread.take().unwrap().join().unwrap(); diff --git a/src/lib.rs b/src/lib.rs index acd96ee5f..fdab5e92a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -89,3 +89,4 @@ pub use heapsizeof::*; pub use squeeze::*; pub use semantic_version::*; pub use network::*; +pub use io::*; diff --git a/src/network/host.rs b/src/network/host.rs index d61ef614c..8917b1f8e 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -1,5 +1,5 @@ use std::mem; -use std::net::{SocketAddr }; +use std::net::{SocketAddr}; use std::collections::{HashMap}; use std::hash::{Hasher}; use std::str::{FromStr}; @@ -103,20 +103,20 @@ impl Encodable for CapabilityInfo { } /// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. -pub struct NetworkContext<'s, Message> where Message: Send + 'static { - io: IoContext<'s, NetworkIoMessage>, - protocol: Option, +pub struct NetworkContext<'s, 'io, Message> where Message: Send + 'static, 'io: 's { + io: &'s mut IoContext<'io, NetworkIoMessage>, + protocol: ProtocolId, connections: &'s mut Slab, timers: &'s mut HashMap, session: Option, } -impl<'s, Message> NetworkContext<'s, Message> where Message: Send + 'static, { +impl<'s, 'io, Message> NetworkContext<'s, 'io, Message> where Message: Send + 'static, { /// Create a new network IO access point. Takes references to all the data that can be updated within the IO handler. - fn new(io: IoContext<'s, NetworkIoMessage>, - protocol: Option, + fn new(io: &'s mut IoContext<'io, NetworkIoMessage>, + protocol: ProtocolId, session: Option, connections: &'s mut Slab, - timers: &'s mut HashMap) -> NetworkContext<'s, Message> { + timers: &'s mut HashMap) -> NetworkContext<'s, 'io, Message> { NetworkContext { io: io, protocol: protocol, @@ -126,15 +126,11 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + 'static, { } } - fn set_protocol(&mut self, protocol: ProtocolId) { - self.protocol = Some(protocol); - } - /// Send a packet over the network to another peer. pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { match self.connections.get_mut(Token(peer)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.send_packet(self.protocol.unwrap(), packet_id as u8, &data).unwrap_or_else(|e| { + s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { warn!(target: "net", "Send error: {:?}", e); }); //TODO: don't copy vector data }, @@ -164,7 +160,7 @@ impl<'s, Message> NetworkContext<'s, Message> where Message: Send + 'static, { pub fn register_timer(&mut self, ms: u64) -> Result{ match self.io.register_timer(ms) { Ok(token) => { - self.timers.insert(token, self.protocol.unwrap()); + self.timers.insert(token, self.protocol); Ok(token) }, e @ Err(_) => e, @@ -259,7 +255,7 @@ impl Host where Message: Send { } } - fn maintain_network(&mut self, io: IoContext>) { + fn maintain_network(&mut self, io: &mut IoContext>) { self.connect_peers(io); } @@ -271,7 +267,7 @@ impl Host where Message: Send { self.connections.iter().any(|e| match e { &ConnectionEntry::Handshake(ref h) => h.id.eq(&id), _ => false }) } - fn connect_peers(&mut self, mut io: IoContext>) { + fn connect_peers(&mut self, io: &mut IoContext>) { struct NodeInfo { id: NodeId, peer_type: PeerType @@ -296,7 +292,7 @@ impl Host where Message: Send { for n in to_connect.iter() { if n.peer_type == PeerType::Required { if req_conn < IDEAL_PEERS { - self.connect_peer(&n.id, &mut io); + self.connect_peer(&n.id, io); } req_conn += 1; } @@ -311,7 +307,7 @@ impl Host where Message: Send { for n in to_connect.iter() { if n.peer_type == PeerType::Optional && open_slots > 0 { open_slots -= 1; - self.connect_peer(&n.id, &mut io); + self.connect_peer(&n.id, io); } } } @@ -362,11 +358,11 @@ impl Host where Message: Send { } - fn accept(&mut self, _io: IoContext>) { + fn accept(&mut self, _io: &mut IoContext>) { trace!(target: "net", "accept"); } - fn connection_writable<'s>(&'s mut self, token: StreamToken, io: IoContext<'s, NetworkIoMessage>) { + fn connection_writable<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { let mut kill = false; let mut create_session = false; match self.connections.get_mut(Token(token)) { @@ -387,27 +383,25 @@ impl Host where Message: Send { warn!(target: "net", "Received event for unknown connection"); } } - let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); if kill { - Host::kill_connection(token, &mut net_context, &mut self.handlers); + self.kill_connection(token, io); + return; + } else if create_session { + self.start_session(token, io); } - if create_session { - Host::start_session(&self.info, token, &mut net_context); - } - match net_context.connections.get_mut(Token(token)) { + match self.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.reregister(net_context.io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + s.reregister(io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); }, _ => (), } } - fn connection_closed<'s>(&'s mut self, token: TimerToken, io: IoContext<'s, NetworkIoMessage>) { - let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); - Host::kill_connection(token, &mut net_context, &mut self.handlers); + fn connection_closed<'s>(&'s mut self, token: TimerToken, io: &mut IoContext<'s, NetworkIoMessage>) { + self.kill_connection(token, io); } - fn connection_readable<'s>(&'s mut self, token: StreamToken, io: IoContext<'s, NetworkIoMessage>) { + fn connection_readable<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { let mut kill = false; let mut create_session = false; let mut ready_data: Vec = Vec::new(); @@ -451,37 +445,35 @@ impl Host where Message: Send { warn!(target: "net", "Received event for unknown connection"); } } - let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); if kill { - Host::kill_connection(token, &mut net_context, &mut self.handlers); + self.kill_connection(token, io); + return; } if create_session { - Host::start_session(&self.info, token, &mut net_context); + self.start_session(token, io); } for p in ready_data { let mut h = self.handlers.get_mut(p).unwrap(); - net_context.set_protocol(p); - h.connected(&mut net_context, &token); + h.connected(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token); } if let Some((p, packet_id, data)) = packet_data { let mut h = self.handlers.get_mut(p).unwrap(); - net_context.set_protocol(p); - h.read(&mut net_context, &token, packet_id, &data[1..]); + h.read(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token, packet_id, &data[1..]); } - match net_context.connections.get_mut(Token(token)) { + match self.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Session(ref mut s)) => { - s.reregister(net_context.io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); + s.reregister(io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); }, _ => (), } } - fn start_session(info: &HostInfo, token: StreamToken, io: &mut NetworkContext) { - let event_loop = &mut io.io.event_loop; - io.connections.replace_with(Token(token), |c| { + fn start_session(&mut self, token: StreamToken, io: &mut IoContext>) { + let info = &self.info; + self.connections.replace_with(Token(token), |c| { match c { - ConnectionEntry::Handshake(h) => Session::new(h, event_loop, info) + ConnectionEntry::Handshake(h) => Session::new(h, io.event_loop, info) .map(|s| Some(ConnectionEntry::Session(s))) .unwrap_or_else(|e| { debug!(target: "net", "Session construction error: {:?}", e); @@ -492,17 +484,16 @@ impl Host where Message: Send { }).expect("Error updating slab with session"); } - fn connection_timeout<'s>(&'s mut self, token: StreamToken, io: IoContext<'s, NetworkIoMessage>) { - let mut net_context = NetworkContext::new(io, None, Some(token), &mut self.connections, &mut self.timers); - Host::kill_connection(token, &mut net_context, &mut self.handlers) + fn connection_timeout<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { + self.kill_connection(token, io) } - fn kill_connection(token: StreamToken, io: &mut NetworkContext, handlers: &mut HashMap>>) { + fn kill_connection<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { let mut to_disconnect: Vec = Vec::new(); - match io.connections.get_mut(Token(token)) { + match self.connections.get_mut(Token(token)) { Some(&mut ConnectionEntry::Handshake(_)) => (), // just abandon handshake Some(&mut ConnectionEntry::Session(ref mut s)) if s.is_ready() => { - for (p, _) in handlers.iter_mut() { + for (p, _) in self.handlers.iter_mut() { if s.have_capability(p) { to_disconnect.push(p); } @@ -511,17 +502,16 @@ impl Host where Message: Send { _ => (), } for p in to_disconnect { - let mut h = handlers.get_mut(p).unwrap(); - io.set_protocol(p); - h.disconnected(io, &token); + let mut h = self.handlers.get_mut(p).unwrap(); + h.disconnected(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token); } - io.connections.remove(Token(token)); + self.connections.remove(Token(token)); } } impl IoHandler> for Host where Message: Send + 'static { /// Initialize networking - fn initialize(&mut self, io: IoContext>) { + fn initialize(&mut self, io: &mut IoContext>) { /* match ::ifaces::Interface::get_all().unwrap().into_iter().filter(|x| x.kind == ::ifaces::Kind::Packet && x.addr.is_some()).next() { Some(iface) => config.public_address = iface.addr.unwrap(), @@ -545,7 +535,7 @@ impl IoHandler> for Host where Messa self.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); } - fn stream_hup(&mut self, io: IoContext>, stream: StreamToken) { + fn stream_hup<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { trace!(target: "net", "Hup: {}", stream); match stream { FIRST_CONNECTION ... LAST_CONNECTION => self.connection_closed(stream, io), @@ -553,7 +543,7 @@ impl IoHandler> for Host where Messa }; } - fn stream_readable(&mut self, io: IoContext>, stream: StreamToken) { + fn stream_readable<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { match stream { TCP_ACCEPT => self.accept(io), FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(stream, io), @@ -562,39 +552,41 @@ impl IoHandler> for Host where Messa } } - fn stream_writable(&mut self, io: IoContext>, stream: StreamToken) { + fn stream_writable<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { match stream { FIRST_CONNECTION ... LAST_CONNECTION => self.connection_writable(stream, io), _ => panic!("Received unknown writable token"), } } - fn timeout(&mut self, io: IoContext>, token: TimerToken) { + fn timeout<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, token: TimerToken) { match token { IDLE => self.maintain_network(io), FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, io), NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, _ => { - let protocol: ProtocolId = self.timers.get_mut(&token).expect("Unknown user timer token"); + let protocol = *self.timers.get_mut(&token).expect("Unknown user timer token"); match self.handlers.get_mut(protocol) { None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, Some(h) => { - h.timeout(&mut NetworkContext::new(io, Some(protocol), Some(token), &mut self.connections, &mut self.timers), token); + h.timeout(&mut NetworkContext::new(io, protocol, Some(token), &mut self.connections, &mut self.timers), token); } } } } } - fn message(&mut self, io: IoContext>, message: &mut NetworkIoMessage) { + fn message<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, message: &'s mut NetworkIoMessage) { match message { &mut NetworkIoMessage::AddHandler { ref mut handler, ref protocol, ref versions } => { - self.handlers.insert(protocol, mem::replace(handler, None).unwrap()); + let mut h = mem::replace(handler, None).unwrap(); + h.initialize(&mut NetworkContext::new(io, protocol, None, &mut self.connections, &mut self.timers)); + self.handlers.insert(protocol, h); for v in versions { self.info.capabilities.push(CapabilityInfo { protocol: protocol, version: *v, packet_count:0 }); } @@ -620,11 +612,9 @@ impl IoHandler> for Host where Messa ref protocol, ref message } => { - let mut net_context = NetworkContext::new(io, None, None, &mut self.connections, &mut self.timers); for (p, h) in self.handlers.iter_mut() { if p != protocol { - net_context.set_protocol(p); - h.message(&mut net_context, &message); + h.message(&mut NetworkContext::new(io, p, None, &mut self.connections, &mut self.timers), &message); } } } diff --git a/src/network/mod.rs b/src/network/mod.rs index 1a1450cb6..7bc7ee71c 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -4,38 +4,42 @@ /// /// ```rust /// extern crate ethcore_util as util; -/// use util::network::*; +/// use util::*; /// /// struct MyHandler; /// -/// impl ProtocolHandler for MyHandler { -/// fn initialize(&mut self, io: &mut NetworkContext) { +/// struct MyMessage { +/// data: u32 +/// } +/// +/// impl NetworkProtocolHandler for MyHandler { +/// fn initialize(&mut self, io: &mut NetworkContext) { /// io.register_timer(1000); /// } /// -/// fn read(&mut self, io: &mut NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { +/// fn read(&mut self, io: &mut NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { /// println!("Received {} ({} bytes) from {}", packet_id, data.len(), peer); /// } /// -/// fn connected(&mut self, io: &mut NetworkContext, peer: &PeerId) { +/// fn connected(&mut self, io: &mut NetworkContext, peer: &PeerId) { /// println!("Connected {}", peer); /// } /// -/// fn disconnected(&mut self, io: &mut NetworkContext, peer: &PeerId) { +/// fn disconnected(&mut self, io: &mut NetworkContext, peer: &PeerId) { /// println!("Disconnected {}", peer); /// } /// -/// fn timeout(&mut self, io: &mut NetworkContext, timer: TimerToken) { +/// fn timeout(&mut self, io: &mut NetworkContext, timer: TimerToken) { /// println!("Timeout {}", timer); /// } /// -/// fn message(&mut self, io: &mut NetworkContext, message: &Message) { -/// println!("Message {}:{}", message.protocol, message.id); +/// fn message(&mut self, io: &mut NetworkContext, message: &MyMessage) { +/// println!("Message {}", message.data); /// } /// } /// /// fn main () { -/// let mut service = NetworkService::start().expect("Error creating network service"); +/// let mut service = NetworkService::::start().expect("Error creating network service"); /// service.register_protocol(Box::new(MyHandler), "myproto", &[1u8]); /// /// // Wait for quit condition @@ -43,7 +47,6 @@ /// // Drop the service /// } /// ``` -extern crate mio; mod host; mod connection; mod handshake; @@ -55,7 +58,7 @@ mod node; pub type PeerId = host::PeerId; pub type PacketId = host::PacketId; -pub type NetworkContext<'s, Message> = host::NetworkContext<'s, Message>; +pub type NetworkContext<'s,'io, Message> = host::NetworkContext<'s, 'io, Message>; pub type NetworkService = service::NetworkService; pub type NetworkIoMessage = host::NetworkIoMessage; pub type NetworkError = error::NetworkError; @@ -66,6 +69,8 @@ use io::*; /// All the handler function are called from within IO event loop. /// `Message` is the type for message data. pub trait NetworkProtocolHandler: Send where Message: Send { + /// Initialize the handler + fn initialize(&mut self, _io: &mut NetworkContext) {} /// Called when new network packet received. fn read(&mut self, io: &mut NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]); /// Called when new peer is connected. Only called when peer supports the same protocol. From a5bb7b7f921fab34f78c61882c7eb309c27bd2ba Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 13 Jan 2016 15:08:36 +0100 Subject: [PATCH 329/379] Work around ICE --- src/io/mod.rs | 4 +--- src/io/service.rs | 2 +- src/network/host.rs | 15 ++++++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/io/mod.rs b/src/io/mod.rs index 497e1ae78..86581b2c3 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -70,9 +70,7 @@ pub type TimerToken = service::TimerToken; pub type StreamToken = service::StreamToken; pub type IoContext<'s, M> = service::IoContext<'s, M>; pub type IoService = service::IoService; -pub const USER_TOKEN_START: usize = service::USER_TOKEN; - - +//pub const USER_TOKEN_START: usize = service::USER_TOKEN; // TODO: ICE in rustc 1.7.0-nightly (49c382779 2016-01-12) #[cfg(test)] mod tests { diff --git a/src/io/service.rs b/src/io/service.rs index aab8f5c2d..f3fe9c020 100644 --- a/src/io/service.rs +++ b/src/io/service.rs @@ -13,7 +13,7 @@ pub type StreamToken = usize; const MAX_USER_TIMERS: usize = 32; const USER_TIMER: usize = 0; const LAST_USER_TIMER: usize = USER_TIMER + MAX_USER_TIMERS - 1; -pub const USER_TOKEN: usize = LAST_USER_TIMER + 1; +//const USER_TOKEN: usize = LAST_USER_TIMER + 1; /// Messages used to communicate with the event loop from other threads. pub enum IoMessage where Message: Send + Sized { diff --git a/src/network/host.rs b/src/network/host.rs index 8917b1f8e..a807804c6 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -45,12 +45,13 @@ impl NetworkConfiguration { } // Tokens -const TOKEN_BEGIN: usize = USER_TOKEN_START; -const TCP_ACCEPT: usize = TOKEN_BEGIN; -const IDLE: usize = TOKEN_BEGIN + 1; -const NODETABLE_RECEIVE: usize = TOKEN_BEGIN + 2; -const NODETABLE_MAINTAIN: usize = TOKEN_BEGIN + 3; -const NODETABLE_DISCOVERY: usize = TOKEN_BEGIN + 4; +//const TOKEN_BEGIN: usize = USER_TOKEN_START; // TODO: ICE in rustc 1.7.0-nightly (49c382779 2016-01-12) +const TOKEN_BEGIN: usize = 32; +const TCP_ACCEPT: usize = TOKEN_BEGIN + 1; +const IDLE: usize = TOKEN_BEGIN + 2; +const NODETABLE_RECEIVE: usize = TOKEN_BEGIN + 3; +const NODETABLE_MAINTAIN: usize = TOKEN_BEGIN + 4; +const NODETABLE_DISCOVERY: usize = TOKEN_BEGIN + 5; const FIRST_CONNECTION: usize = TOKEN_BEGIN + 16; const LAST_CONNECTION: usize = FIRST_CONNECTION + MAX_CONNECTIONS - 1; @@ -545,9 +546,9 @@ impl IoHandler> for Host where Messa fn stream_readable<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { match stream { - TCP_ACCEPT => self.accept(io), FIRST_CONNECTION ... LAST_CONNECTION => self.connection_readable(stream, io), NODETABLE_RECEIVE => {}, + TCP_ACCEPT => self.accept(io), _ => panic!("Received unknown readable token"), } } From df3e3edc8a40b959aa21dd1544cdfbae8a6d61f0 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 13 Jan 2016 15:14:24 +0100 Subject: [PATCH 330/379] bytes_ref --- src/bytes.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/bytes.rs b/src/bytes.rs index 8c56d6bf3..e925d4509 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -39,6 +39,7 @@ use std::fmt; use std::slice; use std::cmp::Ordering; use std::error::Error as StdError; +use std::ops::{Deref, DerefMut}; use uint::{U128, U256}; use hash::FixedHash; @@ -89,6 +90,31 @@ impl ToPretty for Bytes { } } +pub enum BytesRef<'a> { + Flexible(&'a mut Bytes), + Fixed(&'a mut [u8]) +} + +impl<'a> Deref for BytesRef<'a> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + match self { + &BytesRef::Flexible(ref bytes) => bytes, + &BytesRef::Fixed(ref bytes) => bytes + } + } +} + +impl <'a> DerefMut for BytesRef<'a> { + fn deref_mut(&mut self) -> &mut [u8] { + match self { + &mut BytesRef::Flexible(ref mut bytes) => bytes, + &mut BytesRef::Fixed(ref mut bytes) => bytes + } + } +} + /// Vector of bytes pub type Bytes = Vec; @@ -436,4 +462,4 @@ fn populate_big_types() { let mut h = h256_from_u64(0x69); h.copy_raw_from(&a); assert_eq!(h, h256_from_hex("ffffffffffffffffffffffffffffffffffffffff000000000000000000000069")); -} \ No newline at end of file +} From 6c44f29b061d478b5f15d97e36bb128576c3fde0 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 13 Jan 2016 17:42:57 +0100 Subject: [PATCH 331/379] json aid changes --- src/json_aid.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/json_aid.rs b/src/json_aid.rs index a382c61ec..c0b5a5180 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -18,8 +18,13 @@ pub fn bytes_from_json(json: &Json) -> Bytes { } pub fn address_from_json(json: &Json) -> Address { - let s = json.as_string().unwrap_or("0000000000000000000000000000000000000000"); - if s.len() % 2 == 1 { + address_from_str(json.as_string().unwrap_or("0000000000000000000000000000000000000000")) +} + +pub fn address_from_str<'a>(s: &'a str) -> Address { + if s.len() == 0 { + Address::new() + } else if s.len() % 2 == 1 { address_from_hex(&("0".to_string() + &(clean(s).to_string()))[..]) } else { address_from_hex(clean(s)) @@ -35,7 +40,7 @@ pub fn h256_from_json(json: &Json) -> H256 { } } -pub fn u256_from_hex(s: &str) -> U256 { +pub fn u256_from_str(s: &str) -> U256 { if s.len() >= 2 && &s[0..2] == "0x" { U256::from_str(&s[2..]).unwrap_or(U256::from(0)) } else { @@ -44,7 +49,7 @@ pub fn u256_from_hex(s: &str) -> U256 { } pub fn u256_from_json(json: &Json) -> U256 { - u256_from_hex(json.as_string().unwrap_or("")) + u256_from_str(json.as_string().unwrap_or("")) } pub fn usize_from_json(json: &Json) -> usize { From dd99ebd85914f61ad16b8504ad854ec96cbe516b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 18:41:33 +0100 Subject: [PATCH 332/379] From::from(u64) for hashes. --- src/hash.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/hash.rs b/src/hash.rs index 5eb76a05e..67d502633 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -377,6 +377,19 @@ macro_rules! impl_hash { pub fn from_bloomed(b: &T) -> Self where T: FixedHash { b.bloom_part($size) } } + + impl From for $from { + fn from(mut value: u64) -> $from { + let mut ret = $from::new(); + for i in 0..8 { + if i < $size { + ret.0[$size - i - 1] = (value & 0xff) as u8; + value >>= 8; + } + } + ret + } + } } } @@ -387,7 +400,7 @@ impl<'a> From<&'a U256> for H256 { value.to_bytes(&mut ret); ret } - } + } } impl From for Address { From 5cc5e7780ce15be0ca315a1f5f1ca1274668e9c2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 18:58:57 +0100 Subject: [PATCH 333/379] Tests and an additional From::from for hex string -> hash that defaults to 0s for bad strings. --- src/hash.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/hash.rs b/src/hash.rs index 67d502633..544e82fd2 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -35,6 +35,14 @@ pub trait FixedHash: Sized + BytesConvertable + Populatable { fn is_zero(&self) -> bool; } +fn clean_0x(s: &str) -> &str { + if s.len() >= 2 && &s[0..2] == "0x" { + &s[2..] + } else { + s + } +} + macro_rules! impl_hash { ($from: ident, $size: expr) => { #[derive(Eq)] @@ -390,6 +398,17 @@ macro_rules! impl_hash { ret } } + + impl<'_> From<&'_ str> for $from { + fn from(s: &'_ str) -> $from { + use std::str::FromStr; + if s.len() % 2 == 1 { + $from::from_str(&("0".to_string() + &(clean_0x(s).to_string()))[..]).unwrap_or($from::new()) + } else { + $from::from_str(clean_0x(s)).unwrap_or($from::new()) + } + } + } } } @@ -518,5 +537,19 @@ mod tests { let a = Address::from(h); assert_eq!(address, a); } + + #[test] + fn from_u64() { + assert_eq!(H128::from(0x1234567890abcdef), H128::from_str("00000000000000001234567890abcdef").unwrap()); + assert_eq!(H64::from(0x1234567890abcdef), H64::from_str("1234567890abcdef").unwrap()); + assert_eq!(H32::from(0x1234567890abcdef), H32::from_str("90abcdef").unwrap()); + } + + #[test] + fn from_str() { + assert_eq!(H64::from(0x1234567890abcdef), H64::from("0x1234567890abcdef")); + assert_eq!(H64::from(0x1234567890abcdef), H64::from("1234567890abcdef")); + assert_eq!(H64::from(0x234567890abcdef), H64::from("0x234567890abcdef")); + } } From 3a1d829b1dbff866ba460c1d4e1e740d368f26c1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 19:00:19 +0100 Subject: [PATCH 334/379] Additional test. --- src/hash.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hash.rs b/src/hash.rs index 544e82fd2..73ffdd538 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -550,6 +550,8 @@ mod tests { assert_eq!(H64::from(0x1234567890abcdef), H64::from("0x1234567890abcdef")); assert_eq!(H64::from(0x1234567890abcdef), H64::from("1234567890abcdef")); assert_eq!(H64::from(0x234567890abcdef), H64::from("0x234567890abcdef")); + // too short. + assert_eq!(H64::from(0), H64::from("0x34567890abcdef")); } } From 90645e30a6db01f2938744f8e8e535c489e5e799 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 13 Jan 2016 22:05:10 +0100 Subject: [PATCH 335/379] update pr to use From::from --- src/json_aid.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/json_aid.rs b/src/json_aid.rs index c0b5a5180..d015bb13c 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -18,17 +18,7 @@ pub fn bytes_from_json(json: &Json) -> Bytes { } pub fn address_from_json(json: &Json) -> Address { - address_from_str(json.as_string().unwrap_or("0000000000000000000000000000000000000000")) -} - -pub fn address_from_str<'a>(s: &'a str) -> Address { - if s.len() == 0 { - Address::new() - } else if s.len() % 2 == 1 { - address_from_hex(&("0".to_string() + &(clean(s).to_string()))[..]) - } else { - address_from_hex(clean(s)) - } + From::from(json.as_string().unwrap_or("0000000000000000000000000000000000000000")) } pub fn h256_from_json(json: &Json) -> H256 { From b23ec6ee69bd8da6fbca82ff7104e68263a24e32 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 23:06:44 +0100 Subject: [PATCH 336/379] Nice macros. --- src/common.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/common.rs b/src/common.rs index 39294155c..c57357af8 100644 --- a/src/common.rs +++ b/src/common.rs @@ -5,3 +5,31 @@ pub use uint::*; pub use bytes::*; pub use vector::*; pub use sha3::*; + +#[macro_export] +macro_rules! map { + ( $( $x:expr => $y:expr ),* ) => { + vec![ $( ($x, $y) ),* ].into_iter().collect::>() + } +} + +#[macro_export] +macro_rules! mapx { + ( $( $x:expr => $y:expr ),* ) => { + vec![ $( ( From::from($x), From::from($y) ) ),* ].into_iter().collect::>() + } +} + +#[macro_export] +macro_rules! x { + ( $x:expr ) => { + From::from($x) + } +} + +#[macro_export] +macro_rules! xx { + ( $x:expr ) => { + From::from(From::from($x)) + } +} From 4d2437906effd9f6ebe3e84cad3a29d216f29931 Mon Sep 17 00:00:00 2001 From: arkpar Date: Wed, 13 Jan 2016 23:13:57 +0100 Subject: [PATCH 337/379] Io channel --- src/io/mod.rs | 1 + src/io/service.rs | 18 ++++++++++++++++++ src/network/host.rs | 14 +++----------- src/network/mod.rs | 1 + src/network/service.rs | 5 +++++ 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/io/mod.rs b/src/io/mod.rs index 86581b2c3..23a8509cc 100644 --- a/src/io/mod.rs +++ b/src/io/mod.rs @@ -70,6 +70,7 @@ pub type TimerToken = service::TimerToken; pub type StreamToken = service::StreamToken; pub type IoContext<'s, M> = service::IoContext<'s, M>; pub type IoService = service::IoService; +pub type IoChannel = service::IoChannel; //pub const USER_TOKEN_START: usize = service::USER_TOKEN; // TODO: ICE in rustc 1.7.0-nightly (49c382779 2016-01-12) #[cfg(test)] diff --git a/src/io/service.rs b/src/io/service.rs index f3fe9c020..b8f300670 100644 --- a/src/io/service.rs +++ b/src/io/service.rs @@ -146,6 +146,19 @@ impl Handler for IoManager where Message: Send + 'static { } } +/// Allows sending messages into the event loop. All the IO handlers will get the message +/// in the `message` callback. +pub struct IoChannel where Message: Send { + channel: Sender> +} + +impl IoChannel where Message: Send { + pub fn send(&mut self, message: Message) -> Result<(), IoError> { + try!(self.channel.send(IoMessage::UserMessage(message))); + Ok(()) + } +} + /// General IO Service. Starts an event loop and dispatches IO requests. /// 'Message' is a notification message type pub struct IoService where Message: Send + 'static { @@ -180,6 +193,11 @@ impl IoService where Message: Send + 'static { try!(self.host_channel.send(IoMessage::UserMessage(message))); Ok(()) } + + /// Create a new message channel + pub fn channel(&mut self) -> IoChannel { + IoChannel { channel: self.host_channel.clone() } + } } impl Drop for IoService where Message: Send { diff --git a/src/network/host.rs b/src/network/host.rs index a807804c6..4362f0d53 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -76,10 +76,7 @@ pub enum NetworkIoMessage where Message: Send { data: Vec, }, /// User message - User { - protocol: ProtocolId, - message: Message, - }, + User(Message), } /// Local (temporary) peer session ID. @@ -609,14 +606,9 @@ impl IoHandler> for Host where Messa } } }, - &mut NetworkIoMessage::User { - ref protocol, - ref message - } => { + &mut NetworkIoMessage::User(ref message) => { for (p, h) in self.handlers.iter_mut() { - if p != protocol { - h.message(&mut NetworkContext::new(io, p, None, &mut self.connections, &mut self.timers), &message); - } + h.message(&mut NetworkContext::new(io, p, None, &mut self.connections, &mut self.timers), &message); } } } diff --git a/src/network/mod.rs b/src/network/mod.rs index 7bc7ee71c..a47e88927 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -61,6 +61,7 @@ pub type PacketId = host::PacketId; pub type NetworkContext<'s,'io, Message> = host::NetworkContext<'s, 'io, Message>; pub type NetworkService = service::NetworkService; pub type NetworkIoMessage = host::NetworkIoMessage; +pub use network::host::NetworkIoMessage::User as UserMessage; pub type NetworkError = error::NetworkError; use io::*; diff --git a/src/network/service.rs b/src/network/service.rs index be70e2ee1..401c74634 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -40,5 +40,10 @@ impl NetworkService where Message: Send + 'static { })); Ok(()) } + + pub fn io(&mut self) -> &mut IoService> { + &mut self.io_service + } + } From 5fd5c461b3748dae9247de0cbc995c0b3636fc2a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Wed, 13 Jan 2016 23:19:08 +0100 Subject: [PATCH 338/379] u256_from_ hex -> str. --- src/json_aid.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/json_aid.rs b/src/json_aid.rs index a382c61ec..aca998a14 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -35,7 +35,7 @@ pub fn h256_from_json(json: &Json) -> H256 { } } -pub fn u256_from_hex(s: &str) -> U256 { +pub fn u256_from_str(s: &str) -> U256 { if s.len() >= 2 && &s[0..2] == "0x" { U256::from_str(&s[2..]).unwrap_or(U256::from(0)) } else { @@ -44,7 +44,7 @@ pub fn u256_from_hex(s: &str) -> U256 { } pub fn u256_from_json(json: &Json) -> U256 { - u256_from_hex(json.as_string().unwrap_or("")) + u256_from_str(json.as_string().unwrap_or("")) } pub fn usize_from_json(json: &Json) -> usize { From 46810e961c95726ef52b52fb0b41c1828a5d09c1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 01:27:02 +0100 Subject: [PATCH 339/379] Broken casting. --- src/hash.rs | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 73ffdd538..de63ab061 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -412,8 +412,18 @@ macro_rules! impl_hash { } } -impl<'a> From<&'a U256> for H256 { - fn from(value: &'a U256) -> H256 { +impl From for H256 { + fn from(value: U256) -> H256 { + unsafe { + let mut ret: H256 = ::std::mem::uninitialized(); + value.to_bytes(&mut ret); + ret + } + } +} + +impl<'_> From<&'_ U256> for H256 { + fn from(value: &'_ U256) -> H256 { unsafe { let mut ret: H256 = ::std::mem::uninitialized(); value.to_bytes(&mut ret); @@ -432,6 +442,16 @@ impl From for Address { } } +impl<'_> From<&'_ H256> for Address { + fn from(value: &'_ H256) -> Address { + unsafe { + let mut ret: Address = ::std::mem::uninitialized(); + ::std::ptr::copy(value.as_ptr().offset(12), ret.as_mut_ptr(), 20); + ret + } + } +} + impl From
for H256 { fn from(value: Address) -> H256 { unsafe { @@ -442,6 +462,16 @@ impl From
for H256 { } } +impl<'_> From<&'_ Address> for H256 { + fn from(value: &'_ Address) -> H256 { + unsafe { + let mut ret = H256::new(); + ::std::ptr::copy(value.as_ptr(), ret.as_mut_ptr().offset(12), 20); + ret + } + } +} + pub fn h256_from_hex(s: &str) -> H256 { use std::str::FromStr; H256::from_str(s).unwrap() From 787a119f120e59277f94d6c16dc03cda79db49db Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 01:28:09 +0100 Subject: [PATCH 340/379] Workaround. --- src/hash.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index de63ab061..5d8e8a0fe 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -441,7 +441,7 @@ impl From for Address { } } } - +/* impl<'_> From<&'_ H256> for Address { fn from(value: &'_ H256) -> Address { unsafe { @@ -451,7 +451,7 @@ impl<'_> From<&'_ H256> for Address { } } } - +*/ impl From
for H256 { fn from(value: Address) -> H256 { unsafe { From 1c89b9732da7f3cb1f47f316d7e49f0fb72ba71d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 12:27:41 +0100 Subject: [PATCH 341/379] Nicer display output for hash types. --- src/hash.rs | 5 ++++- src/json_aid.rs | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/hash.rs b/src/hash.rs index 5d8e8a0fe..0d26d74e6 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -206,7 +206,10 @@ macro_rules! impl_hash { } impl fmt::Display for $from { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - (self as &fmt::Debug).fmt(f) + for i in self.0[0..3].iter() { + try!(write!(f, "{:02x}", i)); + } + write!(f, "…{:02x}", self.0.last().unwrap()) } } diff --git a/src/json_aid.rs b/src/json_aid.rs index aca998a14..a50c84c5e 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -35,6 +35,10 @@ pub fn h256_from_json(json: &Json) -> H256 { } } +pub fn vec_h256_from_json(json: &Json) -> Vec { + json.as_array().unwrap().iter().map(&h256_from_json).collect() +} + pub fn u256_from_str(s: &str) -> U256 { if s.len() >= 2 && &s[0..2] == "0x" { U256::from_str(&s[2..]).unwrap_or(U256::from(0)) From d63db8523cd59ca5413046a44fc432a7da34a61b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 13:10:18 +0100 Subject: [PATCH 342/379] Fix test, revert fn hex() behaviour. --- src/hash.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 0d26d74e6..f67833072 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -383,7 +383,7 @@ macro_rules! impl_hash { } impl $from { pub fn hex(&self) -> String { - format!("{}", self) + format!("{:?}", self) } pub fn from_bloomed(b: &T) -> Self where T: FixedHash { b.bloom_part($size) } @@ -520,7 +520,7 @@ mod tests { fn hash() { let h = H64([0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]); assert_eq!(H64::from_str("0123456789abcdef").unwrap(), h); - assert_eq!(format!("{}", h), "0123456789abcdef"); + assert_eq!(format!("{}", h), "012345…ef"); assert_eq!(format!("{:?}", h), "0123456789abcdef"); assert_eq!(h.hex(), "0123456789abcdef"); assert!(h == h); From a2f13e1efb7fe30401c8aa4cfbcf1d759ce48998 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 14 Jan 2016 16:52:10 +0100 Subject: [PATCH 343/379] Minor fixes --- src/io/service.rs | 1 + src/network/host.rs | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/io/service.rs b/src/io/service.rs index b8f300670..1512b958e 100644 --- a/src/io/service.rs +++ b/src/io/service.rs @@ -136,6 +136,7 @@ impl Handler for IoManager where Message: Send + 'static { handler, } => { self.handlers.push(handler); + self.handlers.last_mut().unwrap().initialize(&mut IoContext::new(event_loop, &mut self.timers)); }, IoMessage::UserMessage(ref mut data) => { for h in self.handlers.iter_mut() { diff --git a/src/network/host.rs b/src/network/host.rs index 4362f0d53..7a43b6959 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -23,6 +23,8 @@ const _DEFAULT_PORT: u16 = 30304; const MAX_CONNECTIONS: usize = 1024; const IDEAL_PEERS: u32 = 10; +const MAINTENANCE_TIMEOUT: u64 = 1000; + #[derive(Debug)] struct NetworkConfiguration { listen_address: SocketAddr, @@ -164,6 +166,18 @@ impl<'s, 'io, Message> NetworkContext<'s, 'io, Message> where Message: Send + 's e @ Err(_) => e, } } + + /// Returns peer identification string + pub fn peer_info(&self, peer: PeerId) -> String { + match self.connections.get(Token(peer)) { + Some(&ConnectionEntry::Session(ref s)) => { + s.info.client_version.clone() + }, + _ => { + "unknown".to_string() + } + } + } } /// Shared host information @@ -255,6 +269,7 @@ impl Host where Message: Send { fn maintain_network(&mut self, io: &mut IoContext>) { self.connect_peers(io); + io.event_loop.timeout_ms(Token(IDLE), MAINTENANCE_TIMEOUT).unwrap(); } fn have_session(&self, id: &NodeId) -> bool { @@ -340,6 +355,7 @@ impl Host where Message: Send { let nonce = self.info.next_nonce(); match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(token, id, socket, &nonce).expect("Can't create handshake"))) { Some(token) => { + warn!(target: "slab", "inserted {}", token.as_usize()); match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.start(&self.info, true) @@ -469,6 +485,21 @@ impl Host where Message: Send { fn start_session(&mut self, token: StreamToken, io: &mut IoContext>) { let info = &self.info; + // TODO: use slab::replace_with (currently broken) + match self.connections.remove(Token(token)) { + Some(ConnectionEntry::Handshake(h)) => { + match Session::new(h, io.event_loop, info) { + Ok(session) => { + assert!(Token(token) == self.connections.insert(ConnectionEntry::Session(session)).ok().unwrap()); + }, + Err(e) => { + debug!(target: "net", "Session construction error: {:?}", e); + } + } + }, + _ => panic!("Error updating slab with session") + } + /* self.connections.replace_with(Token(token), |c| { match c { ConnectionEntry::Handshake(h) => Session::new(h, io.event_loop, info) @@ -480,6 +511,7 @@ impl Host where Message: Send { _ => { panic!("No handshake to create a session from"); } } }).expect("Error updating slab with session"); + */ } fn connection_timeout<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { @@ -504,6 +536,7 @@ impl Host where Message: Send { h.disconnected(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token); } self.connections.remove(Token(token)); + warn!(target: "slab", "removed {}", token); } } @@ -518,7 +551,7 @@ impl IoHandler> for Host where Messa // Start listening for incoming connections io.event_loop.register(&self.listener, Token(TCP_ACCEPT), EventSet::readable(), PollOpt::edge()).unwrap(); - io.event_loop.timeout_ms(Token(IDLE), 1000).unwrap(); //TODO: check delay + io.event_loop.timeout_ms(Token(IDLE), MAINTENANCE_TIMEOUT).unwrap(); // open the udp socket io.event_loop.register(&self.udp_socket, Token(NODETABLE_RECEIVE), EventSet::readable(), PollOpt::edge()).unwrap(); io.event_loop.timeout_ms(Token(NODETABLE_MAINTAIN), 7200).unwrap(); From db25f7e590984900484c4ab8f2a79f6818e48ec8 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 14 Jan 2016 16:56:59 +0100 Subject: [PATCH 344/379] Minor fixes --- src/hash.rs | 10 ++++++++++ src/network/host.rs | 2 -- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index f67833072..e3c05a79a 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -444,6 +444,16 @@ impl From for Address { } } } + +impl From for H64 { + fn from(value: H256) -> H64 { + unsafe { + let mut ret: H64 = ::std::mem::uninitialized(); + ::std::ptr::copy(value.as_ptr().offset(20), ret.as_mut_ptr(), 8); + ret + } + } +} /* impl<'_> From<&'_ H256> for Address { fn from(value: &'_ H256) -> Address { diff --git a/src/network/host.rs b/src/network/host.rs index 7a43b6959..b562ca765 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -355,7 +355,6 @@ impl Host where Message: Send { let nonce = self.info.next_nonce(); match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(token, id, socket, &nonce).expect("Can't create handshake"))) { Some(token) => { - warn!(target: "slab", "inserted {}", token.as_usize()); match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.start(&self.info, true) @@ -536,7 +535,6 @@ impl Host where Message: Send { h.disconnected(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token); } self.connections.remove(Token(token)); - warn!(target: "slab", "removed {}", token); } } From df2e9854c7524cb72cc396ab74fae81bc65de836 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 14 Jan 2016 19:04:13 +0100 Subject: [PATCH 345/379] Host info --- src/network/host.rs | 2 +- src/network/service.rs | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index b562ca765..abe1cbfcc 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -223,7 +223,7 @@ enum ConnectionEntry { /// Root IO handler. Manages protocol handlers, IO timers and network connections. pub struct Host where Message: Send { - info: HostInfo, + pub info: HostInfo, udp_socket: UdpSocket, listener: TcpListener, connections: Slab, diff --git a/src/network/service.rs b/src/network/service.rs index 401c74634..48cc11152 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -8,15 +8,19 @@ use io::*; /// `Message` defines a notification data type. pub struct NetworkService where Message: Send + 'static { io_service: IoService>, + host_info: String, } impl NetworkService where Message: Send + 'static { /// Starts IO event loop pub fn start() -> Result, UtilError> { let mut io_service = try!(IoService::>::start()); - try!(io_service.register_handler(Box::new(Host::new()))); + let host = Box::new(Host::new()); + let host_info = host.info.client_version.clone(); + try!(io_service.register_handler(host)); Ok(NetworkService { - io_service: io_service + io_service: io_service, + host_info: host_info, }) } @@ -41,9 +45,16 @@ impl NetworkService where Message: Send + 'static { Ok(()) } + /// Returns host identifier string as advertised to other peers + pub fn host_info(&self) -> String { + self.host_info.clone() + } + + /// Returns underlying io service. pub fn io(&mut self) -> &mut IoService> { &mut self.io_service } + } From 4f53db60ed1b163db6d4a0e1490002d235d277f5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 21:24:03 +0100 Subject: [PATCH 346/379] New JSON conversion traits. --- Cargo.toml | 1 + src/bytes.rs | 2 +- src/common.rs | 1 + src/hash.rs | 34 +++++++++---- src/json_aid.rs | 101 ++++++++++++++++++++++++++++++++------ src/lib.rs | 10 ++-- src/sha3.rs | 8 +-- src/standard.rs | 4 +- src/uint.rs | 128 ++++++++++++++++++++++++++++++++---------------- 9 files changed, 213 insertions(+), 76 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9da1b73c9..6e32edcab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } rust-crypto = "0.2.34" elastic-array = "0.4" heapsize = "0.2" +itertools = "0.4" [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/bytes.rs b/src/bytes.rs index e925d4509..479a91df0 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -40,7 +40,7 @@ use std::slice; use std::cmp::Ordering; use std::error::Error as StdError; use std::ops::{Deref, DerefMut}; -use uint::{U128, U256}; +use uint::{Uint, U128, U256}; use hash::FixedHash; pub struct PrettySlice<'a> (&'a [u8]); diff --git a/src/common.rs b/src/common.rs index c57357af8..78f9fbf64 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,4 +1,5 @@ pub use standard::*; +pub use from_json::*; pub use error::*; pub use hash::*; pub use uint::*; diff --git a/src/hash.rs b/src/hash.rs index f67833072..8c1772b2c 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -1,23 +1,18 @@ //! General hash types, a fixed-size raw-data type used as the output of hash functions. -use std::str::FromStr; -use std::fmt; -use std::ops; -use std::hash::{Hash, Hasher}; -use std::ops::{Index, IndexMut, Deref, DerefMut, BitOr, BitOrAssign, BitAnd, BitXor}; -use std::cmp::{PartialOrd, Ordering}; -use rustc_serialize::hex::*; +use standard::*; +use math::log2; use error::UtilError; use rand::Rng; use rand::os::OsRng; use bytes::{BytesConvertable,Populatable}; -use math::log2; -use uint::U256; +use from_json::*; +use uint::{Uint, U256}; /// Trait for a fixed-size byte array to be used as the output of hash functions. /// /// Note: types implementing `FixedHash` must be also `BytesConvertable`. -pub trait FixedHash: Sized + BytesConvertable + Populatable { +pub trait FixedHash: Sized + BytesConvertable + Populatable + FromStr + Default { fn new() -> Self; /// Synonym for `new()`. Prefer to new as it's more readable. fn zero() -> Self; @@ -196,6 +191,20 @@ macro_rules! impl_hash { } } + impl FromJson for $from { + fn from_json(json: &Json) -> Self { + match json { + &Json::String(ref s) => { + match s.len() % 2 { + 0 => FromStr::from_str(clean_0x(s)).unwrap(), + _ => FromStr::from_str(&("0".to_string() + &(clean_0x(s).to_string()))[..]).unwrap() + } + }, + _ => Default::default(), + } + } + } + impl fmt::Debug for $from { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for i in self.0.iter() { @@ -381,6 +390,7 @@ macro_rules! impl_hash { &self ^ &rhs } } + impl $from { pub fn hex(&self) -> String { format!("{:?}", self) @@ -389,6 +399,10 @@ macro_rules! impl_hash { pub fn from_bloomed(b: &T) -> Self where T: FixedHash { b.bloom_part($size) } } + impl Default for $from { + fn default() -> Self { $from::new() } + } + impl From for $from { fn from(mut value: u64) -> $from { let mut ret = $from::new(); diff --git a/src/json_aid.rs b/src/json_aid.rs index d065352af..570f466bc 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -8,6 +8,14 @@ pub fn clean(s: &str) -> &str { } } +pub fn u256_from_str(s: &str) -> U256 { + if s.len() >= 2 && &s[0..2] == "0x" { + U256::from_str(&s[2..]).unwrap_or(U256::from(0)) + } else { + U256::from_dec_str(s).unwrap_or(U256::from(0)) + } +} + pub fn bytes_from_json(json: &Json) -> Bytes { let s = json.as_string().unwrap_or(""); if s.len() % 2 == 1 { @@ -34,34 +42,97 @@ pub fn vec_h256_from_json(json: &Json) -> Vec { json.as_array().unwrap().iter().map(&h256_from_json).collect() } -pub fn u256_from_str(s: &str) -> U256 { - if s.len() >= 2 && &s[0..2] == "0x" { - U256::from_str(&s[2..]).unwrap_or(U256::from(0)) - } else { - U256::from_dec_str(s).unwrap_or(U256::from(0)) - } -} - -pub fn u256_from_json(json: &Json) -> U256 { - u256_from_str(json.as_string().unwrap_or("")) +pub fn map_h256_h256_from_json(json: &Json) -> BTreeMap { + json.as_object().unwrap().iter().fold(BTreeMap::new(), |mut m, (key, value)| { + m.insert(H256::from(&u256_from_str(key)), H256::from(&U256::from_json(value))); + m + }) } pub fn usize_from_json(json: &Json) -> usize { - u256_from_json(json).low_u64() as usize + U256::from_json(json).low_u64() as usize } pub fn u64_from_json(json: &Json) -> u64 { - u256_from_json(json).low_u64() + U256::from_json(json).low_u64() } pub fn u32_from_json(json: &Json) -> u32 { - u256_from_json(json).low_u32() + U256::from_json(json).low_u32() } pub fn u16_from_json(json: &Json) -> u16 { - u256_from_json(json).low_u32() as u16 + U256::from_json(json).low_u32() as u16 } pub fn u8_from_json(json: &Json) -> u8 { - u256_from_json(json).low_u32() as u8 + U256::from_json(json).low_u32() as u8 +} + +impl FromJson for Vec where T: FromJson { + fn from_json(json: &Json) -> Self { + json.as_array().unwrap().iter().map(|x|T::from_json(x)).collect() + } +} + +impl FromJson for u64 { + fn from_json(json: &Json) -> Self { + U256::from_json(json).low_u64() + } +} + +impl FromJson for u32 { + fn from_json(json: &Json) -> Self { + U256::from_json(json).low_u64() as u32 + } +} + +impl FromJson for u16 { + fn from_json(json: &Json) -> Self { + U256::from_json(json).low_u64() as u16 + } +} + +impl FromJson for u8 { + fn from_json(json: &Json) -> Self { + U256::from_json(json).low_u64() as u8 + } +} + +#[test] +fn u256_from_json() { + let j = Json::from_str("{ \"dec\": \"10\", \"hex\": \"0x0a\", \"int\": 10 }").unwrap(); + + let v: U256 = xjson!(&j["dec"]); + assert_eq!(U256::from(10), v); + let v: U256 = xjson!(&j["hex"]); + assert_eq!(U256::from(10), v); + let v: U256 = xjson!(&j["int"]); + assert_eq!(U256::from(10), v); +} + +#[test] +fn h256_from_json_() { + let j = Json::from_str("{ \"with\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"without\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\" }").unwrap(); + + let v: H256 = xjson!(&j["with"]); + assert_eq!(H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(), v); + let v: H256 = xjson!(&j["without"]); + assert_eq!(H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(), v); +} + +#[test] +fn vec_u256_from_json() { + let j = Json::from_str("{ \"array\": [ \"10\", \"0x0a\", 10] }").unwrap(); + + let v: Vec = xjson!(&j["array"]); + assert_eq!(vec![U256::from(10); 3], v); +} + +#[test] +fn vec_h256_from_json_() { + let j = Json::from_str("{ \"array\": [ \"1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\"] }").unwrap(); + + let v: Vec = xjson!(&j["array"]); + assert_eq!(vec![H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(); 2], v); } diff --git a/src/lib.rs b/src/lib.rs index 7bfe147f9..533ae86bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,16 +34,15 @@ extern crate mio; extern crate rand; extern crate rocksdb; extern crate tiny_keccak; - #[macro_use] extern crate heapsize; #[macro_use] extern crate log; #[macro_use] extern crate lazy_static; - +#[macro_use] +extern crate itertools; extern crate env_logger; - extern crate time; extern crate crypto as rcrypto; extern crate secp256k1; @@ -51,12 +50,16 @@ extern crate arrayvec; extern crate elastic_array; pub mod standard; +#[macro_use] +pub mod from_json; +#[macro_use] pub mod common; pub mod error; pub mod hash; pub mod uint; pub mod bytes; pub mod rlp; +pub mod misc; pub mod json_aid; pub mod vector; pub mod sha3; @@ -75,6 +78,7 @@ pub mod semantic_version; pub mod network; pub use common::*; +pub use misc::*; pub use json_aid::*; pub use rlp::*; pub use hashdb::*; diff --git a/src/sha3.rs b/src/sha3.rs index 5b0a0c6a4..c23dfc7a9 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -5,6 +5,8 @@ use tiny_keccak::Keccak; use bytes::{BytesConvertable,Populatable}; use hash::{H256, FixedHash}; +pub const SHA3_EMPTY: H256 = H256( [0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70] ); + /// Types implementing this trait are sha3able. /// /// ``` @@ -43,12 +45,10 @@ impl Hashable for T where T: BytesConvertable { #[test] fn sha3_empty() { - use std::str::FromStr; - assert_eq!([0u8; 0].sha3(), H256::from_str("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap()); + assert_eq!([0u8; 0].sha3(), SHA3_EMPTY); } #[test] fn sha3_as() { - use std::str::FromStr; - assert_eq!([0x41u8; 32].sha3(), H256::from_str("59cad5948673622c1d64e2322488bf01619f7ff45789741b15a9f782ce9290a8").unwrap()); + assert_eq!([0x41u8; 32].sha3(), From::from("59cad5948673622c1d64e2322488bf01619f7ff45789741b15a9f782ce9290a8")); } diff --git a/src/standard.rs b/src/standard.rs index 7298dc232..19a084a2c 100644 --- a/src/standard.rs +++ b/src/standard.rs @@ -7,6 +7,7 @@ pub use std::ptr; pub use std::result; pub use std::option; pub use std::mem; +pub use std::ops; pub use std::path::Path; pub use std::str::{FromStr}; @@ -22,6 +23,7 @@ pub use std::collections::*; pub use rustc_serialize::json::Json; pub use rustc_serialize::base64::FromBase64; -pub use rustc_serialize::hex::FromHex; +pub use rustc_serialize::hex::{FromHex, FromHexError}; pub use heapsize::HeapSizeOf; +pub use itertools::Itertools; diff --git a/src/uint.rs b/src/uint.rs index b5607390b..78e8d7cd0 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -21,17 +21,8 @@ ///! The functions here are designed to be fast. ///! -use std::fmt; -use std::cmp::*; -use std::ops::*; -use std::str::FromStr; -use std::hash::{Hash, Hasher}; -use rustc_serialize::hex::{FromHex, FromHexError}; - -pub trait FromDecStr: Sized { - type Err; - fn from_dec_str(value: &str) -> Result; -} +use standard::*; +use from_json::*; macro_rules! impl_map_from { ($thing:ident, $from:ty, $to:ty) => { @@ -43,32 +34,77 @@ macro_rules! impl_map_from { } } +pub trait Uint: Sized + Default + FromStr + From + FromJson + fmt::Debug + fmt::Display + PartialOrd + Ord + PartialEq + Eq + Hash { + + /// Size of this type. + const SIZE: usize; + + fn zero() -> Self; + fn one() -> Self; + + type FromDecStrErr; + fn from_dec_str(value: &str) -> Result; + + /// Conversion to u32 + fn low_u32(&self) -> u32; + + /// Conversion to u64 + fn low_u64(&self) -> u64; + + /// Conversion to u32 with overflow checking + fn as_u32(&self) -> u32; + + /// Conversion to u64 with overflow checking + fn as_u64(&self) -> u64; + + /// Return the least number of bits needed to represent the number + fn bits(&self) -> usize; + fn bit(&self, index: usize) -> bool; + fn byte(&self, index: usize) -> u8; + fn to_bytes(&self, bytes: &mut[u8]); + + fn exp10(n: usize) -> Self; +} + macro_rules! construct_uint { ($name:ident, $n_words:expr) => ( /// Little-endian large integer type #[derive(Copy, Clone, Eq, PartialEq)] pub struct $name(pub [u64; $n_words]); - impl $name { - pub const SIZE: usize = $n_words * 8; + impl Uint for $name { + const SIZE: usize = $n_words * 8; + + type FromDecStrErr = FromHexError; + + /// TODO: optimize, throw appropriate err + fn from_dec_str(value: &str) -> Result { + Ok(value.bytes() + .map(|b| b - 48) + .fold($name::from(0u64), | acc, c | + // fast multiplication by 10 + // (acc << 3) + (acc << 1) => acc * 10 + (acc << 3) + (acc << 1) + $name::from(c) + )) + } /// Conversion to u32 #[inline] - pub fn low_u32(&self) -> u32 { + fn low_u32(&self) -> u32 { let &$name(ref arr) = self; arr[0] as u32 } /// Conversion to u64 #[inline] - pub fn low_u64(&self) -> u64 { + fn low_u64(&self) -> u64 { let &$name(ref arr) = self; arr[0] } /// Conversion to u32 with overflow checking #[inline] - pub fn as_u32(&self) -> u32 { + fn as_u32(&self) -> u32 { let &$name(ref arr) = self; if (arr[0] & (0xffffffffu64 << 32)) != 0 { panic!("Integer overflow when casting U256") @@ -78,7 +114,7 @@ macro_rules! construct_uint { /// Conversion to u64 with overflow checking #[inline] - pub fn as_u64(&self) -> u64 { + fn as_u64(&self) -> u64 { let &$name(ref arr) = self; for i in 1..$n_words { if arr[i] != 0 { @@ -89,7 +125,7 @@ macro_rules! construct_uint { } /// Return the least number of bits needed to represent the number #[inline] - pub fn bits(&self) -> usize { + fn bits(&self) -> usize { let &$name(ref arr) = self; for i in 1..$n_words { if arr[$n_words - i] > 0 { return (0x40 * ($n_words - i + 1)) - arr[$n_words - i].leading_zeros() as usize; } @@ -98,18 +134,18 @@ macro_rules! construct_uint { } #[inline] - pub fn bit(&self, index: usize) -> bool { + fn bit(&self, index: usize) -> bool { let &$name(ref arr) = self; arr[index / 64] & (1 << (index % 64)) != 0 } #[inline] - pub fn byte(&self, index: usize) -> u8 { + fn byte(&self, index: usize) -> u8 { let &$name(ref arr) = self; (arr[index / 8] >> ((index % 8)) * 8) as u8 } - pub fn to_bytes(&self, bytes: &mut[u8]) { + fn to_bytes(&self, bytes: &mut[u8]) { assert!($n_words * 8 == bytes.len()); let &$name(ref arr) = self; for i in 0..bytes.len() { @@ -120,7 +156,7 @@ macro_rules! construct_uint { } #[inline] - pub fn exp10(n: usize) -> $name { + fn exp10(n: usize) -> $name { match n { 0 => $name::from(1u64), _ => $name::exp10(n - 1) * $name::from(10u64) @@ -128,15 +164,17 @@ macro_rules! construct_uint { } #[inline] - pub fn zero() -> $name { + fn zero() -> $name { From::from(0u64) } #[inline] - pub fn one() -> $name { + fn one() -> $name { From::from(1u64) } + } + impl $name { /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -154,6 +192,12 @@ macro_rules! construct_uint { } } + impl Default for $name { + fn default() -> Self { + $name::zero() + } + } + impl From for $name { fn from(value: u64) -> $name { let mut ret = [0; $n_words]; @@ -162,6 +206,23 @@ macro_rules! construct_uint { } } + impl FromJson for $name { + fn from_json(json: &Json) -> Self { + match json { + &Json::String(ref s) => { + if s.len() >= 2 && &s[0..2] == "0x" { + FromStr::from_str(&s[2..]).unwrap_or(Default::default()) + } else { + Uint::from_dec_str(s).unwrap_or(Default::default()) + } + }, + &Json::U64(u) => From::from(u), + &Json::I64(i) => From::from(i as u64), + _ => Uint::zero(), + } + } + } + impl_map_from!($name, u8, u64); impl_map_from!($name, u16, u64); impl_map_from!($name, u32, u64); @@ -443,22 +504,6 @@ macro_rules! construct_uint { state.finish(); } } - - impl FromDecStr for $name { - type Err = FromHexError; - - /// TODO: optimize, throw appropriate err - fn from_dec_str(value: &str) -> Result { - Ok(value.bytes() - .map(|b| b - 48) - .fold($name::from(0u64), | acc, c | - // fast multiplication by 10 - // (acc << 3) + (acc << 1) => acc * 10 - (acc << 3) + (acc << 1) + $name::from(c) - )) - } - } - ); } @@ -557,8 +602,7 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); #[cfg(test)] mod tests { - use uint::U256; - use uint::FromDecStr; + use uint::{Uint, U256}; use std::str::FromStr; #[test] From ccab2ea9205f55e5c42de4f8b65a6e1174b6b384 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 21:24:16 +0100 Subject: [PATCH 347/379] Missing files for previous commit. --- src/from_json.rs | 12 ++++++++++++ src/misc.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 src/from_json.rs create mode 100644 src/misc.rs diff --git a/src/from_json.rs b/src/from_json.rs new file mode 100644 index 000000000..71a3f50e3 --- /dev/null +++ b/src/from_json.rs @@ -0,0 +1,12 @@ +use standard::*; + +#[macro_export] +macro_rules! xjson { + ( $x:expr ) => { + FromJson::from_json($x) + } +} + +pub trait FromJson { + fn from_json(json: &Json) -> Self; +} diff --git a/src/misc.rs b/src/misc.rs new file mode 100644 index 000000000..e5efd33bb --- /dev/null +++ b/src/misc.rs @@ -0,0 +1,31 @@ +use common::*; + +#[derive(Debug,Clone,PartialEq,Eq)] +/// Diff type for specifying a change (or not). +pub enum Diff where T: Eq { + Same, + Born(T), + Changed(T, T), + Died(T), +} + +impl Diff where T: Eq { + /// Construct new object with given `pre` and `post`. + pub fn new(pre: T, post: T) -> Self { if pre == post { Diff::Same } else { Diff::Changed(pre, post) } } + + /// Get the before value, if there is one. + pub fn pre(&self) -> Option<&T> { match self { &Diff::Died(ref x) | &Diff::Changed(ref x, _) => Some(x), _ => None } } + + /// Get the after value, if there is one. + pub fn post(&self) -> Option<&T> { match self { &Diff::Born(ref x) | &Diff::Changed(_, ref x) => Some(x), _ => None } } + + /// Determine whether there was a change or not. + pub fn is_same(&self) -> bool { match self { &Diff::Same => true, _ => false }} +} + +#[derive(PartialEq,Eq,Clone,Copy)] +/// Boolean type for clean/dirty status. +pub enum Filth { + Clean, + Dirty, +} From 5e6632ca47d5937cf8a88d09b928f9f428c312f1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 21:58:55 +0100 Subject: [PATCH 348/379] Remove cruft --- src/json_aid.rs | 69 +++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 54 deletions(-) diff --git a/src/json_aid.rs b/src/json_aid.rs index 570f466bc..6c9925ffc 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -8,7 +8,7 @@ pub fn clean(s: &str) -> &str { } } -pub fn u256_from_str(s: &str) -> U256 { +fn u256_from_str(s: &str) -> U256 { if s.len() >= 2 && &s[0..2] == "0x" { U256::from_str(&s[2..]).unwrap_or(U256::from(0)) } else { @@ -16,59 +16,26 @@ pub fn u256_from_str(s: &str) -> U256 { } } -pub fn bytes_from_json(json: &Json) -> Bytes { - let s = json.as_string().unwrap_or(""); - if s.len() % 2 == 1 { - FromHex::from_hex(&("0".to_string() + &(clean(s).to_string()))[..]).unwrap_or(vec![]) - } else { - FromHex::from_hex(clean(s)).unwrap_or(vec![]) +impl FromJson for Bytes { + fn from_json(json: &Json) -> Self { + let s = json.as_string().unwrap_or(""); + if s.len() % 2 == 1 { + FromHex::from_hex(&("0".to_string() + &(clean(s).to_string()))[..]).unwrap_or(vec![]) + } else { + FromHex::from_hex(clean(s)).unwrap_or(vec![]) + } } } -pub fn address_from_json(json: &Json) -> Address { - From::from(json.as_string().unwrap_or("0000000000000000000000000000000000000000")) -} - -pub fn h256_from_json(json: &Json) -> H256 { - let s = json.as_string().unwrap_or("0000000000000000000000000000000000000000000000000000000000000000"); - if s.len() % 2 == 1 { - h256_from_hex(&("0".to_string() + &(clean(s).to_string()))[..]) - } else { - h256_from_hex(clean(s)) +impl FromJson for BTreeMap { + fn from_json(json: &Json) -> Self { + json.as_object().unwrap().iter().fold(BTreeMap::new(), |mut m, (key, value)| { + m.insert(x!(&u256_from_str(key)), x!(&U256::from_json(value))); + m + }) } } -pub fn vec_h256_from_json(json: &Json) -> Vec { - json.as_array().unwrap().iter().map(&h256_from_json).collect() -} - -pub fn map_h256_h256_from_json(json: &Json) -> BTreeMap { - json.as_object().unwrap().iter().fold(BTreeMap::new(), |mut m, (key, value)| { - m.insert(H256::from(&u256_from_str(key)), H256::from(&U256::from_json(value))); - m - }) -} - -pub fn usize_from_json(json: &Json) -> usize { - U256::from_json(json).low_u64() as usize -} - -pub fn u64_from_json(json: &Json) -> u64 { - U256::from_json(json).low_u64() -} - -pub fn u32_from_json(json: &Json) -> u32 { - U256::from_json(json).low_u32() -} - -pub fn u16_from_json(json: &Json) -> u16 { - U256::from_json(json).low_u32() as u16 -} - -pub fn u8_from_json(json: &Json) -> u8 { - U256::from_json(json).low_u32() as u8 -} - impl FromJson for Vec where T: FromJson { fn from_json(json: &Json) -> Self { json.as_array().unwrap().iter().map(|x|T::from_json(x)).collect() @@ -93,12 +60,6 @@ impl FromJson for u16 { } } -impl FromJson for u8 { - fn from_json(json: &Json) -> Self { - U256::from_json(json).low_u64() as u8 - } -} - #[test] fn u256_from_json() { let j = Json::from_str("{ \"dec\": \"10\", \"hex\": \"0x0a\", \"int\": 10 }").unwrap(); From b1e986cbdd4cd880cc357b652cba333c1698bf90 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Thu, 14 Jan 2016 22:02:10 +0100 Subject: [PATCH 349/379] Fixing multiplication in uints --- src/uint.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index b5607390b..f25cd77d0 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -145,10 +145,15 @@ macro_rules! construct_uint { for i in 0..$n_words { let upper = other as u64 * (arr[i] >> 32); let lower = other as u64 * (arr[i] & 0xFFFFFFFF); - if i < 3 { - carry[i + 1] += upper >> 32; + + ret[i] = lower.wrapping_add(upper << 32); + + if i < $n_words - 1 { + carry[i + 1] = upper >> 32; + if ret[i] < lower { + carry[i + 1] += 1; + } } - ret[i] = lower + (upper << 32); } $name(ret) + $name(carry) } @@ -557,7 +562,9 @@ pub const BAD_U256: U256 = U256([0xffffffffffffffffu64; 4]); #[cfg(test)] mod tests { + use uint::U128; use uint::U256; + use uint::U512; use uint::FromDecStr; use std::str::FromStr; @@ -737,6 +744,60 @@ mod tests { assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); } + #[test] + pub fn uint128_add() { + assert_eq!( + U128::from_str("fffffffffffffffff").unwrap() + U128::from_str("fffffffffffffffff").unwrap(), + U128::from_str("1ffffffffffffffffe").unwrap() + ); + } + + #[test] + pub fn uint128_add_overflow() { + assert_eq!( + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() + + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap(), + U128::from_str("fffffffffffffffffffffffffffffffe").unwrap() + ); + } + + #[test] + pub fn uint128_mul() { + assert_eq!( + U128::from_str("fffffffff").unwrap() * U128::from_str("fffffffff").unwrap(), + U128::from_str("ffffffffe000000001").unwrap()); + } + + #[test] + pub fn uint512_mul() { + assert_eq!( + U512::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + * + U512::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), + U512::from_str("3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000000000000000000000000000000000000000000000000001").unwrap() + ); + } + + #[test] + pub fn uint256_mul_overflow() { + assert_eq!( + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + * + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), + U256::from_str("1").unwrap() + ); + } + + #[test] + pub fn uint256_mul_overflow2() { + assert_eq!( + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + * + U256::from_str("2").unwrap(), + U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe").unwrap() + ); + } + #[test] fn uint256_div() { assert_eq!(U256::from(10u64) / U256::from(1u64), U256::from(10u64)); From d2cc3b9b5ba7b5af683fbffe2ca981ab9c7d1088 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 22:16:41 +0100 Subject: [PATCH 350/379] Remove unnecessary unwraps in json_aid. --- src/json_aid.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/json_aid.rs b/src/json_aid.rs index 6c9925ffc..e25e9406d 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -18,27 +18,31 @@ fn u256_from_str(s: &str) -> U256 { impl FromJson for Bytes { fn from_json(json: &Json) -> Self { - let s = json.as_string().unwrap_or(""); - if s.len() % 2 == 1 { - FromHex::from_hex(&("0".to_string() + &(clean(s).to_string()))[..]).unwrap_or(vec![]) - } else { - FromHex::from_hex(clean(s)).unwrap_or(vec![]) + match json { + &Json::String(ref s) => match s.len() % 2 { + 0 => FromHex::from_hex(clean(s)).unwrap_or(vec![]), + _ => FromHex::from_hex(&("0".to_string() + &(clean(s).to_string()))[..]).unwrap_or(vec![]), + }, + _ => vec![], } } } impl FromJson for BTreeMap { fn from_json(json: &Json) -> Self { - json.as_object().unwrap().iter().fold(BTreeMap::new(), |mut m, (key, value)| { - m.insert(x!(&u256_from_str(key)), x!(&U256::from_json(value))); - m - }) + match json { + &Json::Object(ref o) => o.iter().map(|(key, value)| (x!(&u256_from_str(key)), x!(&U256::from_json(value)))).collect(), + _ => BTreeMap::new(), + } } } impl FromJson for Vec where T: FromJson { fn from_json(json: &Json) -> Self { - json.as_array().unwrap().iter().map(|x|T::from_json(x)).collect() + match json { + &Json::Array(ref o) => o.iter().map(|x|T::from_json(x)).collect(), + _ => Vec::new(), + } } } From b781fe67532629f21ae52168bad8a5e4675d337d Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 23:13:28 +0100 Subject: [PATCH 351/379] from_existing works for an empty DB if passed empty RLP SHA3. --- src/trie/triedbmut.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index 03d2b59de..d205dc8e0 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -71,6 +71,9 @@ impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { + if !db.exists(root) && root == &SHA3_NULL_RLP { + *root = db.insert(&NULL_RLP); + } assert!(db.exists(root)); TrieDBMut { db: db, From 6a7c823862d557f8994ad62a1d5cacbd42b66d0a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 14 Jan 2016 23:39:59 +0100 Subject: [PATCH 352/379] Option types in JSON, more tests. --- src/hash.rs | 1 + src/json_aid.rs | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index 8c1772b2c..c17b706db 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -195,6 +195,7 @@ macro_rules! impl_hash { fn from_json(json: &Json) -> Self { match json { &Json::String(ref s) => { + println!("s: {}", s); match s.len() % 2 { 0 => FromStr::from_str(clean_0x(s)).unwrap(), _ => FromStr::from_str(&("0".to_string() + &(clean_0x(s).to_string()))[..]).unwrap() diff --git a/src/json_aid.rs b/src/json_aid.rs index e25e9406d..79a71cac6 100644 --- a/src/json_aid.rs +++ b/src/json_aid.rs @@ -46,6 +46,16 @@ impl FromJson for Vec where T: FromJson { } } +impl FromJson for Option where T: FromJson { + fn from_json(json: &Json) -> Self { + match json { + &Json::String(ref o) if o.is_empty() => None, + &Json::Null => None, + _ => Some(FromJson::from_json(json)), + } + } +} + impl FromJson for u64 { fn from_json(json: &Json) -> Self { U256::from_json(json).low_u64() @@ -77,7 +87,7 @@ fn u256_from_json() { } #[test] -fn h256_from_json_() { +fn h256_from_json() { let j = Json::from_str("{ \"with\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"without\": \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\" }").unwrap(); let v: H256 = xjson!(&j["with"]); @@ -95,9 +105,33 @@ fn vec_u256_from_json() { } #[test] -fn vec_h256_from_json_() { +fn vec_h256_from_json() { let j = Json::from_str("{ \"array\": [ \"1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\", \"0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef\"] }").unwrap(); let v: Vec = xjson!(&j["array"]); assert_eq!(vec![H256::from_str("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef").unwrap(); 2], v); } + +#[test] +fn simple_types() { + let j = Json::from_str("{ \"null\": null, \"empty\": \"\", \"int\": 42, \"dec\": \"42\", \"hex\": \"0x2a\" }").unwrap(); + let v: u16 = xjson!(&j["int"]); + assert_eq!(42u16, v); + let v: u32 = xjson!(&j["dec"]); + assert_eq!(42u32, v); + let v: u64 = xjson!(&j["hex"]); + assert_eq!(42u64, v); +} + +#[test] +fn option_types() { + let j = Json::from_str("{ \"null\": null, \"empty\": \"\", \"int\": 42, \"dec\": \"42\", \"hex\": \"0x2a\" }").unwrap(); + let v: Option = xjson!(&j["int"]); + assert_eq!(Some(42u16), v); + let v: Option = xjson!(&j["dec"]); + assert_eq!(Some(42u16), v); + let v: Option = xjson!(&j["null"]); + assert_eq!(None, v); + let v: Option = xjson!(&j["empty"]); + assert_eq!(None, v); +} \ No newline at end of file From 2d36062794e1bc3d92959c12ce8e308ad1d3be32 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 15 Jan 2016 00:50:48 +0100 Subject: [PATCH 353/379] Slab bug workaround --- Cargo.toml | 1 + src/lib.rs | 1 + src/network/host.rs | 47 +++++++++++++++++++++++++----------------- src/network/session.rs | 1 + 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6e32edcab..b7dca6e1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ rust-crypto = "0.2.34" elastic-array = "0.4" heapsize = "0.2" itertools = "0.4" +slab = { git = "https://github.com/arkpar/slab.git" } [dev-dependencies] json-tests = { path = "json-tests" } diff --git a/src/lib.rs b/src/lib.rs index 4b0108c6f..fca71a100 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,7 @@ //! sudo make install //! ``` +extern crate slab; extern crate rustc_serialize; extern crate mio; extern crate rand; diff --git a/src/network/host.rs b/src/network/host.rs index abe1cbfcc..6b3dfbfd1 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -4,7 +4,6 @@ use std::collections::{HashMap}; use std::hash::{Hasher}; use std::str::{FromStr}; use mio::*; -use mio::util::{Slab}; use mio::tcp::*; use mio::udp::*; use hash::*; @@ -17,6 +16,9 @@ use error::*; use io::*; use network::NetworkProtocolHandler; use network::node::*; +use slab::Index; + +type Slab = ::slab::Slab; const _DEFAULT_PORT: u16 = 30304; @@ -128,7 +130,7 @@ impl<'s, 'io, Message> NetworkContext<'s, 'io, Message> where Message: Send + 's /// Send a packet over the network to another peer. pub fn send(&mut self, peer: PeerId, packet_id: PacketId, data: Vec) -> Result<(), UtilError> { - match self.connections.get_mut(Token(peer)) { + match self.connections.get_mut(peer) { Some(&mut ConnectionEntry::Session(ref mut s)) => { s.send_packet(self.protocol, packet_id as u8, &data).unwrap_or_else(|e| { warn!(target: "net", "Send error: {:?}", e); @@ -169,7 +171,7 @@ impl<'s, 'io, Message> NetworkContext<'s, 'io, Message> where Message: Send + 's /// Returns peer identification string pub fn peer_info(&self, peer: PeerId) -> String { - match self.connections.get(Token(peer)) { + match self.connections.get(peer) { Some(&ConnectionEntry::Session(ref s)) => { s.info.client_version.clone() }, @@ -251,7 +253,7 @@ impl Host where Message: Send { }, udp_socket: udp_socket, listener: listener, - connections: Slab::new_starting_at(Token(FIRST_CONNECTION), MAX_CONNECTIONS), + connections: Slab::new_starting_at(FIRST_CONNECTION, MAX_CONNECTIONS), timers: HashMap::new(), nodes: HashMap::new(), handlers: HashMap::new(), @@ -353,8 +355,9 @@ impl Host where Message: Send { }; let nonce = self.info.next_nonce(); - match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(token, id, socket, &nonce).expect("Can't create handshake"))) { + match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(Token(token), id, socket, &nonce).expect("Can't create handshake"))) { Some(token) => { + warn!(target: "slab", "inserted {}", token.as_usize()); match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.start(&self.info, true) @@ -378,7 +381,7 @@ impl Host where Message: Send { fn connection_writable<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { let mut kill = false; let mut create_session = false; - match self.connections.get_mut(Token(token)) { + match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.writable(io.event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Handshake write error: {:?}", e); @@ -402,7 +405,7 @@ impl Host where Message: Send { } else if create_session { self.start_session(token, io); } - match self.connections.get_mut(Token(token)) { + match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Session(ref mut s)) => { s.reregister(io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); }, @@ -419,7 +422,7 @@ impl Host where Message: Send { let mut create_session = false; let mut ready_data: Vec = Vec::new(); let mut packet_data: Option<(ProtocolId, PacketId, Vec)> = None; - match self.connections.get_mut(Token(token)) { + match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.readable(io.event_loop, &self.info).unwrap_or_else(|e| { debug!(target: "net", "Handshake read error: {:?}", e); @@ -474,7 +477,7 @@ impl Host where Message: Send { h.read(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token, packet_id, &data[1..]); } - match self.connections.get_mut(Token(token)) { + match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Session(ref mut s)) => { s.reregister(io.event_loop).unwrap_or_else(|e| debug!(target: "net", "Session registration error: {:?}", e)); }, @@ -485,11 +488,12 @@ impl Host where Message: Send { fn start_session(&mut self, token: StreamToken, io: &mut IoContext>) { let info = &self.info; // TODO: use slab::replace_with (currently broken) - match self.connections.remove(Token(token)) { + /* + match self.connections.remove(token) { Some(ConnectionEntry::Handshake(h)) => { match Session::new(h, io.event_loop, info) { Ok(session) => { - assert!(Token(token) == self.connections.insert(ConnectionEntry::Session(session)).ok().unwrap()); + assert!(token == self.connections.insert(ConnectionEntry::Session(session)).ok().unwrap()); }, Err(e) => { debug!(target: "net", "Session construction error: {:?}", e); @@ -497,9 +501,9 @@ impl Host where Message: Send { } }, _ => panic!("Error updating slab with session") - } - /* - self.connections.replace_with(Token(token), |c| { + }*/ + warn!(target: "slab", "replaced {}", token.as_usize()); + self.connections.replace_with(token, |c| { match c { ConnectionEntry::Handshake(h) => Session::new(h, io.event_loop, info) .map(|s| Some(ConnectionEntry::Session(s))) @@ -510,7 +514,6 @@ impl Host where Message: Send { _ => { panic!("No handshake to create a session from"); } } }).expect("Error updating slab with session"); - */ } fn connection_timeout<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { @@ -519,7 +522,8 @@ impl Host where Message: Send { fn kill_connection<'s>(&'s mut self, token: StreamToken, io: &mut IoContext<'s, NetworkIoMessage>) { let mut to_disconnect: Vec = Vec::new(); - match self.connections.get_mut(Token(token)) { + let mut remove = true; + match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(_)) => (), // just abandon handshake Some(&mut ConnectionEntry::Session(ref mut s)) if s.is_ready() => { for (p, _) in self.handlers.iter_mut() { @@ -528,13 +532,18 @@ impl Host where Message: Send { } } }, - _ => (), + _ => { + remove = false; + }, } for p in to_disconnect { let mut h = self.handlers.get_mut(p).unwrap(); h.disconnected(&mut NetworkContext::new(io, p, Some(token), &mut self.connections, &mut self.timers), &token); } - self.connections.remove(Token(token)); + if remove { + self.connections.remove(token); + warn!(target: "slab", "removed {}", token); + } } } @@ -626,7 +635,7 @@ impl IoHandler> for Host where Messa ref protocol, ref data, } => { - match self.connections.get_mut(Token(*peer as usize)) { + match self.connections.get_mut(*peer as usize) { Some(&mut ConnectionEntry::Session(ref mut s)) => { s.send_packet(protocol, *packet_id as u8, &data).unwrap_or_else(|e| { warn!(target: "net", "Send error: {:?}", e); diff --git a/src/network/session.rs b/src/network/session.rs index 2d8ba2245..828e4b062 100644 --- a/src/network/session.rs +++ b/src/network/session.rs @@ -242,6 +242,7 @@ impl Session { i += 1; } trace!(target: "net", "Hello: {} v{} {} {:?}", client_version, protocol, id, caps); + self.info.client_version = client_version; self.info.capabilities = caps; if protocol != host.protocol_version { return Err(From::from(self.disconnect(DisconnectReason::UselessPeer))); From 825f3733fd5efdb0c3ab1a1699827e1b0488cade Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 15 Jan 2016 00:52:21 +0100 Subject: [PATCH 354/379] Removed debug output --- src/network/host.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index 6b3dfbfd1..9c6b8a557 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -357,7 +357,6 @@ impl Host where Message: Send { let nonce = self.info.next_nonce(); match self.connections.insert_with(|token| ConnectionEntry::Handshake(Handshake::new(Token(token), id, socket, &nonce).expect("Can't create handshake"))) { Some(token) => { - warn!(target: "slab", "inserted {}", token.as_usize()); match self.connections.get_mut(token) { Some(&mut ConnectionEntry::Handshake(ref mut h)) => { h.start(&self.info, true) @@ -502,7 +501,6 @@ impl Host where Message: Send { }, _ => panic!("Error updating slab with session") }*/ - warn!(target: "slab", "replaced {}", token.as_usize()); self.connections.replace_with(token, |c| { match c { ConnectionEntry::Handshake(h) => Session::new(h, io.event_loop, info) @@ -542,7 +540,6 @@ impl Host where Message: Send { } if remove { self.connections.remove(token); - warn!(target: "slab", "removed {}", token); } } } From 8dd4b3689afe74a47db3d7eb07edc2057ecd56bb Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 15 Jan 2016 00:54:43 +0100 Subject: [PATCH 355/379] Removed unused import --- src/network/host.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/network/host.rs b/src/network/host.rs index 9c6b8a557..bcee38aad 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -16,7 +16,6 @@ use error::*; use io::*; use network::NetworkProtocolHandler; use network::node::*; -use slab::Index; type Slab = ::slab::Slab; From 9b78a89f966c34900f49446490cc5c3608f30788 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Fri, 15 Jan 2016 01:41:08 +0100 Subject: [PATCH 356/379] Overflow semantics changed --- src/lib.rs | 1 + src/uint.rs | 227 ++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 212 insertions(+), 16 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7bfe147f9..f49e04af9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ #![feature(op_assign_traits)] #![feature(associated_consts)] +#![feature(wrapping)] //! Ethcore-util library //! //! ### Rust version: diff --git a/src/uint.rs b/src/uint.rs index f25cd77d0..e1e2dcd42 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -26,6 +26,7 @@ use std::cmp::*; use std::ops::*; use std::str::FromStr; use std::hash::{Hash, Hasher}; +use std::num::wrapping::OverflowingOps; use rustc_serialize::hex::{FromHex, FromHexError}; pub trait FromDecStr: Sized { @@ -43,6 +44,14 @@ macro_rules! impl_map_from { } } +macro_rules! panic_on_overflow { + ($name:expr) => { + if $name { + panic!("arithmetic operation overflow") + } + } +} + macro_rules! construct_uint { ($name:ident, $n_words:expr) => ( /// Little-endian large integer type @@ -157,6 +166,31 @@ macro_rules! construct_uint { } $name(ret) + $name(carry) } + + /// Overflowing multiplication by u32 + fn overflowing_mul_u32(self, other: u32) -> ($name, bool) { + let $name(ref arr) = self; + let mut carry = [0u64; $n_words]; + let mut ret = [0u64; $n_words]; + let mut overflow = false; + for i in 0..$n_words { + let upper = other as u64 * (arr[i] >> 32); + let lower = other as u64 * (arr[i] & 0xFFFFFFFF); + + ret[i] = lower.wrapping_add(upper << 32); + + if i < $n_words - 1 { + carry[i + 1] = upper >> 32; + if ret[i] < lower { + carry[i + 1] += 1; + } + } else if (upper >> 32) > 0 || ret[i] < lower { + overflow = true + } + } + let (result, add_overflow) = $name(ret).overflowing_add($name(carry)); + (result, add_overflow || overflow) + } } impl From for $name { @@ -214,6 +248,77 @@ macro_rules! construct_uint { } } + impl OverflowingOps for $name { + fn overflowing_add(self, other: $name) -> ($name, bool) { + let $name(ref me) = self; + let $name(ref you) = other; + let mut ret = [0u64; $n_words]; + let mut carry = [0u64; $n_words]; + let mut b_carry = false; + let mut overflow = false; + + for i in 0..$n_words { + ret[i] = me[i].wrapping_add(you[i]); + + if ret[i] < me[i] { + if i < $n_words - 1 { + carry[i + 1] = 1; + b_carry = true; + } else { + overflow = true + } + } + } + if b_carry { + let (ret, add_overflow) = $name(ret).overflowing_add($name(carry)); + (ret, add_overflow || overflow) + } else { + ($name(ret), overflow) + } + } + + fn overflowing_sub(self, other: $name) -> ($name, bool) { + let (res, _overflow) = (!other).overflowing_add(From::from(1u64)); + let (res, _overflow) = self.overflowing_add(res); + (res, self < other) + } + + fn overflowing_mul(self, other: $name) -> ($name, bool) { + let mut res = $name::from(0u64); + let mut overflow = false; + // TODO: be more efficient about this + for i in 0..(2 * $n_words) { + let (v, mul_overflow) = self.overflowing_mul_u32((other >> (32 * i)).low_u32()); + let (new_res, add_overflow) = res.overflowing_add(v << (32 * i)); + res = new_res; + overflow = overflow || mul_overflow || add_overflow; + } + (res, overflow) + } + + fn overflowing_div(self, other: $name) -> ($name, bool) { + (self / other, false) + } + + fn overflowing_rem(self, other: $name) -> ($name, bool) { + (self % other, false) + } + + fn overflowing_neg(self) -> ($name, bool) { + (!self, true) + } + + fn overflowing_shl(self, _shift32: u32) -> ($name, bool) { + // TODO [todr] not used for now + unimplemented!(); + } + + fn overflowing_shr(self, _shift32: u32) -> ($name, bool) { + // TODO [todr] not used for now + unimplemented!(); + } + } + impl Add<$name> for $name { type Output = $name; @@ -224,10 +329,14 @@ macro_rules! construct_uint { let mut carry = [0u64; $n_words]; let mut b_carry = false; for i in 0..$n_words { - ret[i] = me[i].wrapping_add(you[i]); - if i < $n_words - 1 && ret[i] < me[i] { - carry[i + 1] = 1; - b_carry = true; + if i < $n_words - 1 { + ret[i] = me[i].wrapping_add(you[i]); + if ret[i] < me[i] { + carry[i + 1] = 1; + b_carry = true; + } + } else { + ret[i] = me[i] + you[i]; } } if b_carry { $name(ret) + $name(carry) } else { $name(ret) } @@ -239,7 +348,10 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { - self + !other + From::from(1u64) + panic_on_overflow!(self < other); + let (res, _overflow) = (!other).overflowing_add(From::from(1u64)); + let (res, _overflow) = self.overflowing_add(res); + res } } @@ -281,7 +393,8 @@ macro_rules! construct_uint { loop { if sub_copy >= shift_copy { ret[shift / 64] |= 1 << (shift % 64); - sub_copy = sub_copy - shift_copy; + let (copy, _overflow) = sub_copy.overflowing_sub(shift_copy); + sub_copy = copy } shift_copy = shift_copy >> 1; if shift == 0 { break; } @@ -370,7 +483,7 @@ macro_rules! construct_uint { let bit_shift = shift % 64; for i in 0..$n_words { // Shift - if bit_shift < 64 && i + word_shift < $n_words { + if i + word_shift < $n_words { ret[i + word_shift] += original[i] << bit_shift; } // Carry @@ -567,6 +680,7 @@ mod tests { use uint::U512; use uint::FromDecStr; use std::str::FromStr; + use std::num::wrapping::OverflowingOps; #[test] pub fn uint256_from() { @@ -692,7 +806,7 @@ mod tests { let incr = shr + U256::from(1u64); assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); // Subtraction - let sub = incr - init; + let (sub, _of) = incr.overflowing_sub(init); assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); // Multiplication let mult = sub.mul_u32(300); @@ -740,7 +854,7 @@ mod tests { } #[test] - pub fn uint256_mul() { + pub fn uint256_mul1() { assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); } @@ -755,12 +869,22 @@ mod tests { #[test] pub fn uint128_add_overflow() { assert_eq!( - U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() - + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap(), - U128::from_str("fffffffffffffffffffffffffffffffe").unwrap() + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() + .overflowing_add( + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() + ), + (U128::from_str("fffffffffffffffffffffffffffffffe").unwrap(), true) ); } + #[test] + #[should_panic] + pub fn uint128_add_overflow_panic() { + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap() + + + U128::from_str("ffffffffffffffffffffffffffffffff").unwrap(); + } + #[test] pub fn uint128_mul() { assert_eq!( @@ -782,14 +906,85 @@ mod tests { pub fn uint256_mul_overflow() { assert_eq!( U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() - * - U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), - U256::from_str("1").unwrap() + .overflowing_mul( + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + ), + (U256::from_str("1").unwrap(), true) ); } #[test] - pub fn uint256_mul_overflow2() { + #[should_panic] + pub fn uint256_mul_overflow_panic() { + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + * + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(); + } + + #[test] + pub fn uint256_sub_overflow() { + assert_eq!( + U256::from_str("0").unwrap() + .overflowing_sub( + U256::from_str("1").unwrap() + ), + (U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), true) + ); + } + + #[test] + #[should_panic] + pub fn uint256_sub_overflow_panic() { + U256::from_str("0").unwrap() + - + U256::from_str("1").unwrap(); + } + + #[ignore] + #[test] + pub fn uint256_shl_overflow() { + assert_eq!( + U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(4), + (U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0").unwrap(), true) + ); + } + + #[ignore] + #[test] + #[should_panic] + pub fn uint256_shl_overflow2() { + assert_eq!( + U256::from_str("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(4), + (U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0").unwrap(), false) + ); + } + + #[ignore] + #[test] + pub fn uint256_shr_overflow() { + assert_eq!( + U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shr(4), + (U256::from_str("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), true) + ); + } + + #[ignore] + #[test] + pub fn uint256_shr_overflow2() { + assert_eq!( + U256::from_str("fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0").unwrap() + .overflowing_shr(4), + (U256::from_str("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap(), false) + ); + } + + + + #[test] + pub fn uint256_mul() { assert_eq!( U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() * From 838eea61f9d41b035041c535b868db4a06a91234 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 15 Jan 2016 01:57:14 +0100 Subject: [PATCH 357/379] updated to rocksdb wrapper version 0.3 --- Cargo.toml | 2 +- src/overlaydb.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6e32edcab..f92dcca53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ mio = "0.5.0" rand = "0.3.12" time = "0.1.34" tiny-keccak = "1.0" -rocksdb = "0.2" +rocksdb = "0.3" lazy_static = "0.1" eth-secp256k1 = { git = "https://github.com/arkpar/rust-secp256k1.git" } rust-crypto = "0.2.34" diff --git a/src/overlaydb.rs b/src/overlaydb.rs index 5539a8893..929a492ef 100644 --- a/src/overlaydb.rs +++ b/src/overlaydb.rs @@ -10,7 +10,7 @@ use std::ops::*; use std::sync::*; use std::env; use std::collections::HashMap; -use rocksdb::{DB, Writable}; +use rocksdb::{DB, Writable, IteratorMode}; #[derive(Clone)] /// Implementation of the HashDB trait for a disk-backed database with a memory overlay. @@ -138,7 +138,7 @@ impl OverlayDB { impl HashDB for OverlayDB { fn keys(&self) -> HashMap { let mut ret: HashMap = HashMap::new(); - for (key, _) in self.backing.iterator().from_start() { + for (key, _) in self.backing.iterator(IteratorMode::Start) { let h = H256::from_slice(key.deref()); let r = self.payload(&h).unwrap().1; ret.insert(h, r as i32); From 23fbe394081cdde8f8460cf32c40959d48c2f5fa Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 15 Jan 2016 04:02:24 +0100 Subject: [PATCH 358/379] Fix for assumption that empty trie root RLP can always be looked up. --- src/hash.rs | 1 - src/memorydb.rs | 31 +++++++++++++++++++++++++++---- src/trie/triedb.rs | 2 +- src/trie/triedbmut.rs | 12 +++++------- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/hash.rs b/src/hash.rs index c17b706db..8c1772b2c 100644 --- a/src/hash.rs +++ b/src/hash.rs @@ -195,7 +195,6 @@ macro_rules! impl_hash { fn from_json(json: &Json) -> Self { match json { &Json::String(ref s) => { - println!("s: {}", s); match s.len() % 2 { 0 => FromStr::from_str(clean_0x(s)).unwrap(), _ => FromStr::from_str(&("0".to_string() + &(clean_0x(s).to_string()))[..]).unwrap() diff --git a/src/memorydb.rs b/src/memorydb.rs index 680129670..6f2b2e603 100644 --- a/src/memorydb.rs +++ b/src/memorydb.rs @@ -2,6 +2,7 @@ use hash::*; use bytes::*; +use rlp::*; use sha3::*; use hashdb::*; use std::mem; @@ -53,13 +54,15 @@ use std::collections::HashMap; /// ``` pub struct MemoryDB { data: HashMap, + static_null_rlp: (Bytes, i32), } impl MemoryDB { /// Create a new instance of the memory DB. pub fn new() -> MemoryDB { MemoryDB { - data: HashMap::new() + data: HashMap::new(), + static_null_rlp: (vec![0x80u8; 1], 1), } } @@ -98,6 +101,9 @@ impl MemoryDB { /// Even when Some is returned, the data is only guaranteed to be useful /// when the refs > 0. pub fn raw(&self, key: &H256) -> Option<&(Bytes, i32)> { + if key == &SHA3_NULL_RLP { + return Some(&self.static_null_rlp); + } self.data.get(key) } @@ -108,18 +114,23 @@ impl MemoryDB { } pub fn denote(&self, key: &H256, value: Bytes) -> &(Bytes, i32) { - if self.data.get(&key) == None { + if self.raw(key) == None { unsafe { let p = &self.data as *const HashMap as *mut HashMap; (*p).insert(key.clone(), (value, 0)); } } - self.data.get(key).unwrap() + self.raw(key).unwrap() } } +static NULL_RLP_STATIC: [u8; 1] = [0x80; 1]; + impl HashDB for MemoryDB { fn lookup(&self, key: &H256) -> Option<&[u8]> { + if key == &SHA3_NULL_RLP { + return Some(&NULL_RLP_STATIC); + } match self.data.get(key) { Some(&(ref d, rc)) if rc > 0 => Some(d), _ => None @@ -127,10 +138,13 @@ impl HashDB for MemoryDB { } fn keys(&self) -> HashMap { - self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect::>() + self.data.iter().filter_map(|(k, v)| if v.1 != 0 {Some((k.clone(), v.1))} else {None}).collect() } fn exists(&self, key: &H256) -> bool { + if key == &SHA3_NULL_RLP { + return true; + } match self.data.get(key) { Some(&(_, x)) if x > 0 => true, _ => false @@ -138,6 +152,9 @@ impl HashDB for MemoryDB { } fn insert(&mut self, value: &[u8]) -> H256 { + if value == &NULL_RLP { + return SHA3_NULL_RLP.clone(); + } let key = value.sha3(); if match self.data.get_mut(&key) { Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { @@ -154,6 +171,9 @@ impl HashDB for MemoryDB { } fn emplace(&mut self, key: H256, value: Bytes) { + if value == &NULL_RLP { + return; + } match self.data.get_mut(&key) { Some(&mut (ref mut old_value, ref mut rc @ -0x80000000i32 ... 0)) => { *old_value = value; @@ -168,6 +188,9 @@ impl HashDB for MemoryDB { } fn kill(&mut self, key: &H256) { + if key == &SHA3_NULL_RLP { + return; + } if match self.data.get_mut(key) { Some(&mut (_, ref mut x)) => { *x -= 1; false } None => true diff --git a/src/trie/triedb.rs b/src/trie/triedb.rs index ef07fd47d..862bbb96c 100644 --- a/src/trie/triedb.rs +++ b/src/trie/triedb.rs @@ -81,7 +81,7 @@ impl<'db> TrieDB<'db> { let mut ret = self.db.keys(); for (k, v) in Self::to_map(self.keys()).into_iter() { let keycount = *ret.get(&k).unwrap_or(&0); - match keycount == v as i32 { + match keycount <= v as i32 { true => ret.remove(&k), _ => ret.insert(k, keycount - v as i32), }; diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index d205dc8e0..c6f47fa5e 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -64,16 +64,13 @@ impl<'db> TrieDBMut<'db> { }; // set root rlp - *r.root = r.db.insert(&NULL_RLP); + *r.root = SHA3_NULL_RLP.clone(); r } /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { - if !db.exists(root) && root == &SHA3_NULL_RLP { - *root = db.insert(&NULL_RLP); - } assert!(db.exists(root)); TrieDBMut { db: db, @@ -111,7 +108,7 @@ impl<'db> TrieDBMut<'db> { let mut ret = self.db.keys(); for (k, v) in Self::to_map(self.keys()).into_iter() { let keycount = *ret.get(&k).unwrap_or(&0); - match keycount == v as i32 { + match keycount <= v as i32 { true => ret.remove(&k), _ => ret.insert(k, keycount - v as i32), }; @@ -771,8 +768,9 @@ mod tests { assert!(memtrie.db_items_remaining().is_empty()); unpopulate_trie(&mut memtrie, &x); if *memtrie.root() != SHA3_NULL_RLP || !memtrie.db_items_remaining().is_empty() { - println!("TRIE MISMATCH"); + println!("- TRIE MISMATCH"); println!(""); + println!("remaining: {:?}", memtrie.db_items_remaining()); println!("{:?} vs {:?}", memtrie.root(), real); for i in x.iter() { println!("{:?} -> {:?}", i.0.pretty(), i.1.pretty()); @@ -811,7 +809,7 @@ mod tests { let mut t1 = TrieDBMut::new(&mut memdb, &mut root); t1.insert(&[0x01, 0x23], &big_value.to_vec()); t1.insert(&[0x01, 0x34], &big_value.to_vec()); - trace!("keys remaining {:?}", t1.db_items_remaining()); + println!("********************** keys remaining {:?}", t1.db_items_remaining()); assert!(t1.db_items_remaining().is_empty()); let mut memdb2 = MemoryDB::new(); let mut root2 = H256::new(); From 162a1cd8a2dff5e74b8696448fad06dbee20f7e3 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Fri, 15 Jan 2016 11:57:51 +0100 Subject: [PATCH 359/379] Power for uint --- src/uint.rs | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/uint.rs b/src/uint.rs index 038c85458..f644cdb25 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -183,6 +183,64 @@ macro_rules! construct_uint { } impl $name { + + pub fn pow(self, expon: $name) -> $name { + if expon == $name::zero() { + return $name::one() + } + let is_even = |x : &$name| x.low_u64() & 1 == 0; + + let u_one = $name::one(); + let u_two = $name::from(2); + let mut y = u_one; + let mut n = expon; + let mut x = self; + while n > u_one { + if is_even(&n) { + x = x * x; + n = n / u_two; + } else { + y = x * y; + x = x * x; + n = (n - u_one) / u_two; + } + } + x * y + } + + pub fn overflowing_pow(self, expon: $name) -> ($name, bool) { + if expon == $name::zero() { + return ($name::one(), false) + } + let is_even = |x : &$name| x.low_u64() & 1 == 0; + + let u_one = $name::one(); + let u_two = $name::from(2); + let mut y = u_one; + let mut n = expon; + let mut x = self; + let mut overflow = false; + + while n > u_one { + if is_even(&n) { + let (c, mul_overflow) = x.overflowing_mul(x); + x = c; + overflow |= mul_overflow; + n = n / u_two; + } else { + let (new_y, y_overflow) = x.overflowing_mul(y); + let (new_x, x_overflow) = x.overflowing_mul(x); + x = new_x; + y = new_y; + overflow |= y_overflow | x_overflow; + + n = (n - u_one) / u_two; + } + } + let (res, mul_overflow) = x.overflowing_mul(y); + (res, mul_overflow | overflow) + } + /// Multiplication by u32 fn mul_u32(self, other: u32) -> $name { let $name(ref arr) = self; @@ -894,6 +952,33 @@ mod tests { assert_eq!(U256::from(1000u64).mul_u32(50), U256::from(50000u64)); } + #[test] + fn uint256_pow () { + assert_eq!(U256::from(10).pow(U256::from(0)), U256::from(1)); + assert_eq!(U256::from(10).pow(U256::from(1)), U256::from(10)); + assert_eq!(U256::from(10).pow(U256::from(2)), U256::from(100)); + assert_eq!(U256::from(10).pow(U256::from(3)), U256::from(1000)); + assert_eq!(U256::from(10).pow(U256::from(20)), U256::exp10(20)); + } + + #[test] + #[should_panic] + fn uint256_pow_overflow () { + U256::from(2).pow(U256::from(0x001)); + } + + #[test] + fn uint256_overflowing_pow () { + assert_eq!( + U256::from(2).overflowing_pow(U256::from(0xfe)), + (U256::zero(), false) + ); + assert_eq!( + U256::from(2).overflowing_pow(U256::from(0x001)), + (U256::zero(), true) + ); + } + #[test] pub fn uint256_mul1() { assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); From b8a2a16e8392f41c5fca5ff948a6e118ad0e5211 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Fri, 15 Jan 2016 12:35:27 +0100 Subject: [PATCH 360/379] Fixing tests --- src/uint.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index f644cdb25..c37310cd0 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -409,7 +409,7 @@ macro_rules! construct_uint { let (v, mul_overflow) = self.overflowing_mul_u32((other >> (32 * i)).low_u32()); let (new_res, add_overflow) = res.overflowing_add(v << (32 * i)); res = new_res; - overflow = overflow || mul_overflow || add_overflow; + overflow |= mul_overflow | add_overflow; } (res, overflow) } @@ -964,17 +964,17 @@ mod tests { #[test] #[should_panic] fn uint256_pow_overflow () { - U256::from(2).pow(U256::from(0x001)); + U256::from(2).pow(U256::from(0x100)); } #[test] fn uint256_overflowing_pow () { assert_eq!( - U256::from(2).overflowing_pow(U256::from(0xfe)), - (U256::zero(), false) + U256::from(2).overflowing_pow(U256::from(0xff)), + (U256::from_str("8000000000000000000000000000000000000000000000000000000000000000").unwrap(), false) ); assert_eq!( - U256::from(2).overflowing_pow(U256::from(0x001)), + U256::from(2).overflowing_pow(U256::from(0x100)), (U256::zero(), true) ); } From 1136e65f0dfbdc205281f52e9d8f34ede0436841 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 15 Jan 2016 13:11:30 +0100 Subject: [PATCH 361/379] Add flush function. --- src/common.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/common.rs b/src/common.rs index 78f9fbf64..7750d9ea5 100644 --- a/src/common.rs +++ b/src/common.rs @@ -34,3 +34,8 @@ macro_rules! xx { From::from(From::from($x)) } } + +pub fn flush(s: String) { + ::std::io::stdout().write(s.as_bytes()).unwrap(); + ::std::io::stdout().flush().unwrap(); +} From df151ab92c652c354d7b498a98943c08092e02db Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 15 Jan 2016 14:40:51 +0100 Subject: [PATCH 362/379] Make clear when the panic occurs to a missing trie root. --- src/common.rs | 5 ----- src/standard.rs | 5 +++++ src/trie/triedb.rs | 10 +++++----- src/trie/triedbmut.rs | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/common.rs b/src/common.rs index 7750d9ea5..78f9fbf64 100644 --- a/src/common.rs +++ b/src/common.rs @@ -34,8 +34,3 @@ macro_rules! xx { From::from(From::from($x)) } } - -pub fn flush(s: String) { - ::std::io::stdout().write(s.as_bytes()).unwrap(); - ::std::io::stdout().flush().unwrap(); -} diff --git a/src/standard.rs b/src/standard.rs index 19a084a2c..59b6c4c38 100644 --- a/src/standard.rs +++ b/src/standard.rs @@ -27,3 +27,8 @@ pub use rustc_serialize::hex::{FromHex, FromHexError}; pub use heapsize::HeapSizeOf; pub use itertools::Itertools; + +pub fn flush(s: String) { + ::std::io::stdout().write(s.as_bytes()).unwrap(); + ::std::io::stdout().flush().unwrap(); +} diff --git a/src/trie/triedb.rs b/src/trie/triedb.rs index 862bbb96c..bd34e710d 100644 --- a/src/trie/triedb.rs +++ b/src/trie/triedb.rs @@ -1,10 +1,7 @@ -use std::fmt; +use common::*; use hashdb::*; -use hash::*; use nibbleslice::*; -use bytes::*; use rlp::*; -use std::collections::HashMap; use super::trietraits::*; use super::node::*; @@ -44,7 +41,10 @@ impl<'db> TrieDB<'db> { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist pub fn new(db: &'db HashDB, root: &'db H256) -> Self { - assert!(db.exists(root)); + if !db.exists(root) { + flush(format!("Trie root not found {}", root)); + panic!("Trie root not found!"); + } TrieDB { db: db, root: root, diff --git a/src/trie/triedbmut.rs b/src/trie/triedbmut.rs index c6f47fa5e..832b532f8 100644 --- a/src/trie/triedbmut.rs +++ b/src/trie/triedbmut.rs @@ -1,10 +1,7 @@ -use std::fmt; +use common::*; use hashdb::*; -use hash::*; use nibbleslice::*; -use bytes::*; use rlp::*; -use std::collections::HashMap; use super::node::*; use super::journal::*; use super::trietraits::*; @@ -71,7 +68,10 @@ impl<'db> TrieDBMut<'db> { /// Create a new trie with the backing database `db` and `root` /// Panics, if `root` does not exist pub fn from_existing(db: &'db mut HashDB, root: &'db mut H256) -> Self { - assert!(db.exists(root)); + if !db.exists(root) { + flush(format!("Trie root not found {}", root)); + panic!("Trie root not found!"); + } TrieDBMut { db: db, root: root, From 0ea9e132103fdffae1d560fedb7f289f1ad15c2f Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Fri, 15 Jan 2016 14:40:54 +0100 Subject: [PATCH 363/379] Detecting overflows in shl --- src/uint.rs | 171 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 128 insertions(+), 43 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index c37310cd0..ccc66b447 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -35,8 +35,24 @@ macro_rules! impl_map_from { } } +macro_rules! overflowing { + ($op: expr, $overflow: expr) => ( + { + let (overflow_x, overflow_overflow) = $op; + $overflow |= overflow_overflow; + overflow_x + } + ); + ($op: expr) => ( + { + let (overflow_x, _overflow_overflow) = $op; + overflow_x + } + ); +} + macro_rules! panic_on_overflow { - ($name:expr) => { + ($name: expr) => { if $name { panic!("arithmetic operation overflow") } @@ -223,22 +239,16 @@ macro_rules! construct_uint { while n > u_one { if is_even(&n) { - let (c, mul_overflow) = x.overflowing_mul(x); - x = c; - overflow |= mul_overflow; + x = overflowing!(x.overflowing_mul(x), overflow); n = n / u_two; } else { - let (new_y, y_overflow) = x.overflowing_mul(y); - let (new_x, x_overflow) = x.overflowing_mul(x); - x = new_x; - y = new_y; - overflow |= y_overflow | x_overflow; - + y = overflowing!(x.overflowing_mul(y), overflow); + x = overflowing!(x.overflowing_mul(x), overflow); n = (n - u_one) / u_two; } } - let (res, mul_overflow) = x.overflowing_mul(y); - (res, mul_overflow | overflow) + let res = overflowing!(x.overflowing_mul(y), overflow); + (res, overflow) } /// Multiplication by u32 @@ -283,8 +293,11 @@ macro_rules! construct_uint { overflow = true } } - let (result, add_overflow) = $name(ret).overflowing_add($name(carry)); - (result, add_overflow || overflow) + let result = overflowing!( + $name(ret).overflowing_add($name(carry)), + overflow + ); + (result, overflow) } } @@ -383,21 +396,21 @@ macro_rules! construct_uint { carry[i + 1] = 1; b_carry = true; } else { - overflow = true + overflow = true; } } } - if b_carry { - let (ret, add_overflow) = $name(ret).overflowing_add($name(carry)); - (ret, add_overflow || overflow) + if b_carry { + let ret = overflowing!($name(ret).overflowing_add($name(carry)), overflow); + (ret, overflow) } else { ($name(ret), overflow) } } fn overflowing_sub(self, other: $name) -> ($name, bool) { - let (res, _overflow) = (!other).overflowing_add(From::from(1u64)); - let (res, _overflow) = self.overflowing_add(res); + let res = overflowing!((!other).overflowing_add(From::from(1u64))); + let res = overflowing!(self.overflowing_add(res)); (res, self < other) } @@ -406,10 +419,9 @@ macro_rules! construct_uint { let mut overflow = false; // TODO: be more efficient about this for i in 0..(2 * $n_words) { - let (v, mul_overflow) = self.overflowing_mul_u32((other >> (32 * i)).low_u32()); - let (new_res, add_overflow) = res.overflowing_add(v << (32 * i)); - res = new_res; - overflow |= mul_overflow | add_overflow; + let v = overflowing!(self.overflowing_mul_u32((other >> (32 * i)).low_u32()), overflow); + let res2 = overflowing!(v.overflowing_shl(32 * i as u32), overflow); + res = overflowing!(res.overflowing_add(res2), overflow); } (res, overflow) } @@ -426,9 +438,38 @@ macro_rules! construct_uint { (!self, true) } - fn overflowing_shl(self, _shift32: u32) -> ($name, bool) { - // TODO [todr] not used for now - unimplemented!(); + fn overflowing_shl(self, shift32: u32) -> ($name, bool) { + let $name(ref original) = self; + let mut ret = [0u64; $n_words]; + let shift = shift32 as usize; + let word_shift = shift / 64; + let bit_shift = shift % 64; + for i in 0..$n_words { + // Shift + if i + word_shift < $n_words { + ret[i + word_shift] += original[i] << bit_shift; + } + // Carry + if bit_shift > 0 && i + word_shift + 1 < $n_words { + ret[i + word_shift + 1] += original[i] >> (64 - bit_shift); + } + } + // Detecting overflow + let last = $n_words - word_shift - if bit_shift > 0 { 1 } else { 0 }; + let overflow = if bit_shift > 0 { + (original[last] >> (64 - bit_shift)) > 0 + } else if word_shift > 0 { + original[last] > 0 + } else { + false + }; + + for i in last+1..$n_words-1 { + if original[i] > 0 { + return ($name(ret), true); + } + } + ($name(ret), overflow) } fn overflowing_shr(self, _shift32: u32) -> ($name, bool) { @@ -467,9 +508,8 @@ macro_rules! construct_uint { #[inline] fn sub(self, other: $name) -> $name { panic_on_overflow!(self < other); - let (res, _overflow) = (!other).overflowing_add(From::from(1u64)); - let (res, _overflow) = self.overflowing_add(res); - res + let res = overflowing!((!other).overflowing_add(From::from(1u64))); + overflowing!(self.overflowing_add(res)) } } @@ -480,7 +520,10 @@ macro_rules! construct_uint { let mut res = $name::from(0u64); // TODO: be more efficient about this for i in 0..(2 * $n_words) { - res = res + (self.mul_u32((other >> (32 * i)).low_u32()) << (32 * i)); + let v = self.mul_u32((other >> (32 * i)).low_u32()); + let (r, overflow) = v.overflowing_shl(32 * i as u32); + panic_on_overflow!(overflow); + res = res + r; } res } @@ -511,8 +554,7 @@ macro_rules! construct_uint { loop { if sub_copy >= shift_copy { ret[shift / 64] |= 1 << (shift % 64); - let (copy, _overflow) = sub_copy.overflowing_sub(shift_copy); - sub_copy = copy + sub_copy = overflowing!(sub_copy.overflowing_sub(shift_copy)); } shift_copy = shift_copy >> 1; if shift == 0 { break; } @@ -905,7 +947,7 @@ mod tests { let incr = shr + U256::from(1u64); assert_eq!(incr, U256([0x7DDE000000000001u64, 0x0001BD5B7DDFBD5B, 0, 0])); // Subtraction - let (sub, _of) = incr.overflowing_sub(init); + let sub = overflowing!(incr.overflowing_sub(init)); assert_eq!(sub, U256([0x9F30411021524112u64, 0x0001BD5B7DDFBD5A, 0, 0])); // Multiplication let mult = sub.mul_u32(300); @@ -963,16 +1005,16 @@ mod tests { #[test] #[should_panic] - fn uint256_pow_overflow () { + fn uint256_pow_overflow_panic () { U256::from(2).pow(U256::from(0x100)); } #[test] fn uint256_overflowing_pow () { - assert_eq!( - U256::from(2).overflowing_pow(U256::from(0xff)), - (U256::from_str("8000000000000000000000000000000000000000000000000000000000000000").unwrap(), false) - ); + // assert_eq!( + // U256::from(2).overflowing_pow(U256::from(0xff)), + // (U256::from_str("8000000000000000000000000000000000000000000000000000000000000000").unwrap(), false) + // ); assert_eq!( U256::from(2).overflowing_pow(U256::from(0x100)), (U256::zero(), true) @@ -984,6 +1026,16 @@ mod tests { assert_eq!(U256::from(1u64) * U256::from(10u64), U256::from(10u64)); } + #[test] + pub fn uint256_overflowing_mul() { + assert_eq!( + U256::from_str("100000000000000000000000000000000").unwrap().overflowing_mul( + U256::from_str("100000000000000000000000000000000").unwrap() + ), + (U256::zero(), true) + ); + } + #[test] pub fn uint128_add() { assert_eq!( @@ -1036,9 +1088,11 @@ mod tests { U256::from_str("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() ), (U256::from_str("1").unwrap(), true) - ); + ); } + + #[test] #[should_panic] pub fn uint256_mul_overflow_panic() { @@ -1066,7 +1120,6 @@ mod tests { U256::from_str("1").unwrap(); } - #[ignore] #[test] pub fn uint256_shl_overflow() { assert_eq!( @@ -1076,9 +1129,41 @@ mod tests { ); } - #[ignore] #[test] - #[should_panic] + pub fn uint256_shl_overflow_words() { + assert_eq!( + U256::from_str("0000000000000001ffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(64), + (U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000").unwrap(), true) + ); + assert_eq!( + U256::from_str("0000000000000000ffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(64), + (U256::from_str("ffffffffffffffffffffffffffffffffffffffffffffffff0000000000000000").unwrap(), false) + ); + } + + #[test] + pub fn uint256_shl_overflow_words2() { + assert_eq!( + U256::from_str("00000000000000000000000000000001ffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(128), + (U256::from_str("ffffffffffffffffffffffffffffffff00000000000000000000000000000000").unwrap(), true) + ); + assert_eq!( + U256::from_str("00000000000000000000000000000000ffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(128), + (U256::from_str("ffffffffffffffffffffffffffffffff00000000000000000000000000000000").unwrap(), false) + ); + assert_eq!( + U256::from_str("00000000000000000000000000000000ffffffffffffffffffffffffffffffff").unwrap() + .overflowing_shl(129), + (U256::from_str("fffffffffffffffffffffffffffffffe00000000000000000000000000000000").unwrap(), true) + ); + } + + + #[test] pub fn uint256_shl_overflow2() { assert_eq!( U256::from_str("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap() From 34b516b8b0d3c99d79f0d67b0d152663bbac7a5a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 15 Jan 2016 14:42:16 +0100 Subject: [PATCH 364/379] Move flush back to common. --- src/common.rs | 5 +++++ src/standard.rs | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common.rs b/src/common.rs index 78f9fbf64..7750d9ea5 100644 --- a/src/common.rs +++ b/src/common.rs @@ -34,3 +34,8 @@ macro_rules! xx { From::from(From::from($x)) } } + +pub fn flush(s: String) { + ::std::io::stdout().write(s.as_bytes()).unwrap(); + ::std::io::stdout().flush().unwrap(); +} diff --git a/src/standard.rs b/src/standard.rs index 59b6c4c38..19a084a2c 100644 --- a/src/standard.rs +++ b/src/standard.rs @@ -27,8 +27,3 @@ pub use rustc_serialize::hex::{FromHex, FromHexError}; pub use heapsize::HeapSizeOf; pub use itertools::Itertools; - -pub fn flush(s: String) { - ::std::io::stdout().write(s.as_bytes()).unwrap(); - ::std::io::stdout().flush().unwrap(); -} From 2319fd4cecb921d93f0f54a3d184135e263b318c Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 15 Jan 2016 16:19:21 +0100 Subject: [PATCH 365/379] New list of bootnodes --- src/io/service.rs | 3 ++- src/network/host.rs | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/io/service.rs b/src/io/service.rs index 1512b958e..4ccfb2407 100644 --- a/src/io/service.rs +++ b/src/io/service.rs @@ -30,6 +30,7 @@ pub enum IoMessage where Message: Send + Sized { /// IO access point. This is passed to all IO handlers and provides an interface to the IO subsystem. pub struct IoContext<'s, Message> where Message: Send + 'static { timers: &'s mut Slab, + /// Low leve MIO Event loop for custom handler registration. pub event_loop: &'s mut EventLoop>, } @@ -43,7 +44,7 @@ impl<'s, Message> IoContext<'s, Message> where Message: Send + 'static { } /// Register a new IO timer. Returns a new timer token. 'IoHandler::timeout' will be called with the token. - pub fn register_timer(&mut self, ms: u64) -> Result{ + pub fn register_timer(&mut self, ms: u64) -> Result { match self.timers.insert(UserTimer { delay: ms, }) { diff --git a/src/network/host.rs b/src/network/host.rs index bcee38aad..8b1038c8a 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -164,7 +164,7 @@ impl<'s, 'io, Message> NetworkContext<'s, 'io, Message> where Message: Send + 's self.timers.insert(token, self.protocol); Ok(token) }, - e @ Err(_) => e, + e => e, } } @@ -561,12 +561,13 @@ impl IoHandler> for Host where Messa let port = self.info.config.listen_address.port(); self.info.listen_port = port; - self.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); - self.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@gav.ethdev.com:30300"); - self.add_node("enode://e58d5e26b3b630496ec640f2530f3e7fa8a8c7dfe79d9e9c4aac80e3730132b869c852d3125204ab35bb1b1951f6f2d40996c1034fd8c5a69b383ee337f02ddc@gav.ethdev.com:30303"); - self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); - self.add_node("enode://7f25d3eab333a6b98a8b5ed68d962bb22c876ffcd5561fca54e3c2ef27f754df6f7fd7c9b74cc919067abac154fb8e1f8385505954f161ae440abc355855e034@54.207.93.166:30303"); - self.add_node("enode://5374c1bff8df923d3706357eeb4983cd29a63be40a269aaa2296ee5f3b2119a8978c0ed68b8f6fc84aad0df18790417daadf91a4bfbb786a16c9b0a199fa254a@92.51.165.126:30303"); + //self.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); + // GO bootnodes + self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); // IE + self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR + self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG + // ETH/DEV cpp-ethereum (poc-9.ethdev.com) + self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); } fn stream_hup<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { From 3ec294bda2eefe03a4721415773f971111baebf2 Mon Sep 17 00:00:00 2001 From: arkpar Date: Fri, 15 Jan 2016 16:36:08 +0100 Subject: [PATCH 366/379] Missing files --- src/network/error.rs | 41 ++++++++++++++++++++++ src/network/node.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 src/network/error.rs create mode 100644 src/network/node.rs diff --git a/src/network/error.rs b/src/network/error.rs new file mode 100644 index 000000000..d255cb043 --- /dev/null +++ b/src/network/error.rs @@ -0,0 +1,41 @@ +use io::IoError; +use rlp::*; + +#[derive(Debug, Copy, Clone)] +pub enum DisconnectReason +{ + DisconnectRequested, + //TCPError, + //BadProtocol, + UselessPeer, + //TooManyPeers, + //DuplicatePeer, + //IncompatibleProtocol, + //NullIdentity, + //ClientQuit, + //UnexpectedIdentity, + //LocalIdentity, + //PingTimeout, +} + +#[derive(Debug)] +pub enum NetworkError { + Auth, + BadProtocol, + PeerNotFound, + Disconnect(DisconnectReason), + Io(IoError), +} + +impl From for NetworkError { + fn from(_err: DecoderError) -> NetworkError { + NetworkError::Auth + } +} + +impl From for NetworkError { + fn from(err: IoError) -> NetworkError { + NetworkError::Io(err) + } +} + diff --git a/src/network/node.rs b/src/network/node.rs new file mode 100644 index 000000000..5c08e0a66 --- /dev/null +++ b/src/network/node.rs @@ -0,0 +1,83 @@ +use std::net::{SocketAddr, ToSocketAddrs}; +use std::hash::{Hash, Hasher}; +use std::str::{FromStr}; +use hash::*; +use rlp::*; +use time::Tm; +use error::*; + +/// Node public key +pub type NodeId = H512; + +#[derive(Debug)] +/// Noe address info +pub struct NodeEndpoint { + /// IP(V4 or V6) address + pub address: SocketAddr, + /// Address as string (can be host name). + pub address_str: String, + /// Conneciton port. + pub udp_port: u16 +} + +impl NodeEndpoint { + /// Create endpoint from string. Performs name resolution if given a host name. + fn from_str(s: &str) -> Result { + let address = s.to_socket_addrs().map(|mut i| i.next()); + match address { + Ok(Some(a)) => Ok(NodeEndpoint { + address: a, + address_str: s.to_string(), + udp_port: a.port() + }), + Ok(_) => Err(UtilError::AddressResolve(None)), + Err(e) => Err(UtilError::AddressResolve(Some(e))) + } + } +} + +#[derive(PartialEq, Eq, Copy, Clone)] +pub enum PeerType { + Required, + Optional +} + +pub struct Node { + pub id: NodeId, + pub endpoint: NodeEndpoint, + pub peer_type: PeerType, + pub last_attempted: Option, +} + +impl FromStr for Node { + type Err = UtilError; + fn from_str(s: &str) -> Result { + let (id, endpoint) = if &s[0..8] == "enode://" && s.len() > 136 && &s[136..137] == "@" { + (try!(NodeId::from_str(&s[8..136])), try!(NodeEndpoint::from_str(&s[137..]))) + } + else { + (NodeId::new(), try!(NodeEndpoint::from_str(s))) + }; + + Ok(Node { + id: id, + endpoint: endpoint, + peer_type: PeerType::Optional, + last_attempted: None, + }) + } +} + +impl PartialEq for Node { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} +impl Eq for Node { } + +impl Hash for Node { + fn hash(&self, state: &mut H) where H: Hasher { + self.id.hash(state) + } +} + From 3e1add7051e49f4a5e53a50e2ba067702f9b9aa2 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 15 Jan 2016 19:51:48 +0100 Subject: [PATCH 367/379] flush macro --- src/common.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/common.rs b/src/common.rs index 7750d9ea5..3ec02ad9b 100644 --- a/src/common.rs +++ b/src/common.rs @@ -35,7 +35,23 @@ macro_rules! xx { } } +#[macro_export] +macro_rules! flush { + ($($arg:tt)*) => ($crate::flush(format!("{}", format_args!($($arg)*)))); +} + +#[macro_export] +macro_rules! flushln { + ($fmt:expr) => (flush!(concat!($fmt, "\n"))); + ($fmt:expr, $($arg:tt)*) => (flush!(concat!($fmt, "\n"), $($arg)*)); +} + pub fn flush(s: String) { ::std::io::stdout().write(s.as_bytes()).unwrap(); ::std::io::stdout().flush().unwrap(); } + +#[test] +fn test_flush() { + flushln!("hello_world {:?}", 1); +} From e3f1d70353757d5b8395c81953068c3740aff4d3 Mon Sep 17 00:00:00 2001 From: debris Date: Fri, 15 Jan 2016 21:03:38 +0100 Subject: [PATCH 368/379] display trait implemented for rlp --- src/rlp/rlpin.rs | 7 +++++++ src/rlp/untrusted_rlp.rs | 29 +++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/rlp/rlpin.rs b/src/rlp/rlpin.rs index 2179643d1..4cc83b261 100644 --- a/src/rlp/rlpin.rs +++ b/src/rlp/rlpin.rs @@ -1,3 +1,4 @@ +use std::fmt; use rlp::{View, Decodable, DecoderError, UntrustedRlp, PayloadInfo, Prototype}; impl<'a> From> for Rlp<'a> { @@ -15,6 +16,12 @@ pub struct Rlp<'a> { rlp: UntrustedRlp<'a> } +impl<'a> fmt::Display for Rlp<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "{}", self.rlp) + } +} + impl<'a, 'view> View<'a, 'view> for Rlp<'a> where 'a: 'view { type Prototype = Prototype; type PayloadInfo = PayloadInfo; diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index c0e7c4cf7..a5ee7990e 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -1,4 +1,6 @@ use std::cell::Cell; +use std::fmt; +use rustc_serialize::hex::ToHex; use bytes::{FromBytes}; use rlp::{View, Decoder, Decodable, DecoderError}; @@ -63,6 +65,24 @@ impl<'a> Clone for UntrustedRlp<'a> { } } +impl<'a> fmt::Display for UntrustedRlp<'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match self.prototype() { + Ok(Prototype::Null) => write!(f, "null"), + Ok(Prototype::Data(_)) => write!(f, "\"0x{}\"", self.data().unwrap().to_hex()), + Ok(Prototype::List(len)) => { + try!(write!(f, "[")); + for i in 0..len-1 { + try!(write!(f, "{}, ", self.at(i).unwrap())); + } + try!(write!(f, "{}", self.at(len - 1).unwrap())); + write!(f, "]") + }, + Err(err) => write!(f, "{:?}", err) + } + } +} + impl<'a, 'view> View<'a, 'view> for UntrustedRlp<'a> where 'a: 'view { type Prototype = Result; type PayloadInfo = Result; @@ -410,3 +430,12 @@ impl_array_decodable_recursive!( 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 40, 48, 56, 64, 72, 96, 128, 160, 192, 224, ); + +#[test] +fn test_rlp_display() { + use rustc_serialize::hex::FromHex; + let data = "f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".from_hex().unwrap(); + let rlp = UntrustedRlp::new(&data); + println!("{}", rlp); +} + From 5520497e8156d86e21f435f0e976d5fd023cc153 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 15 Jan 2016 21:25:55 +0100 Subject: [PATCH 369/379] Fixed Uint output. --- src/rlp/rlpin.rs | 9 +++++++++ src/uint.rs | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/rlp/rlpin.rs b/src/rlp/rlpin.rs index 2179643d1..821511405 100644 --- a/src/rlp/rlpin.rs +++ b/src/rlp/rlpin.rs @@ -133,3 +133,12 @@ impl<'a, 'view> Iterator for RlpIterator<'a, 'view> { result } } + +#[test] +fn break_it() { + use common::*; + let h: Bytes = FromHex::from_hex("f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap(); + let r: Rlp = Rlp::new(&h); + let u: U256 = r.val_at(1); + assert_eq!(format!("{}", u), "0x10efbef67941f79b2"); +} diff --git a/src/uint.rs b/src/uint.rs index 038c85458..2a18a2fb0 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -597,8 +597,15 @@ macro_rules! construct_uint { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let &$name(ref data) = self; try!(write!(f, "0x")); + let mut latch = false; for ch in data.iter().rev() { - try!(write!(f, "{:02x}", ch)); + for x in 0..16 { + let ch = ch & (15u64 << ((15 - x) * 4) as u64) >> ((15 - x) * 4) as u64; + if !latch { latch = ch != 0 } + if latch { + try!(write!(f, "{:01x}", ch)); + } + } } Ok(()) } @@ -608,8 +615,15 @@ macro_rules! construct_uint { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let &$name(ref data) = self; try!(write!(f, "0x")); + let mut latch = false; for ch in data.iter().rev() { - try!(write!(f, "{:02x}", ch)); + for x in 0..16 { + let nibble = (ch & (15u64 << ((15 - x) * 4) as u64)) >> (((15 - x) * 4) as u64); + if !latch { latch = nibble != 0 } + if latch { + try!(write!(f, "{:x}", nibble)); + } + } } Ok(()) } From d41ad82aa93853744d0f3b115f350b42cf6b1230 Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 16 Jan 2016 00:38:41 +0100 Subject: [PATCH 370/379] uint fmt debug/display is changed to output decimal representation --- src/uint.rs | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index 038c85458..32cbd2cf1 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -595,16 +595,30 @@ macro_rules! construct_uint { impl fmt::Debug for $name { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let &$name(ref data) = self; - try!(write!(f, "0x")); - for ch in data.iter().rev() { - try!(write!(f, "{:02x}", ch)); - } - Ok(()) + fmt::Display::fmt(self, f) } } impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if *self == $name::zero() { + return write!(f, "0"); + } + + let mut s = String::new(); + let mut current = *self; + let ten = $name::from(10); + + while current != $name::zero() { + s = format!("{}{}", (current % ten).low_u32(), s); + current = current / ten; + } + + write!(f, "{}", s) + } + } + + impl fmt::LowerHex for $name { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let &$name(ref data) = self; try!(write!(f, "0x")); @@ -1052,5 +1066,16 @@ mod tests { assert_eq!(U256::from_dec_str("10").unwrap(), U256::from(10u64)); assert_eq!(U256::from_dec_str("1024").unwrap(), U256::from(1024u64)); } + + #[test] + fn display_uint() { + let s = "12345678987654321023456789"; + assert_eq!(format!("{}", U256::from_dec_str(s).unwrap()), s); + } + + #[test] + fn display_uint_zero() { + assert_eq!(format!("{}", U256::from(0)), "0"); + } } From 2899790cd3d21c1ab2aa2a3559a4c25e09e8041b Mon Sep 17 00:00:00 2001 From: debris Date: Sat, 16 Jan 2016 00:42:02 +0100 Subject: [PATCH 371/379] test for rlp display --- src/rlp/untrusted_rlp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rlp/untrusted_rlp.rs b/src/rlp/untrusted_rlp.rs index a5ee7990e..2bf33ba68 100644 --- a/src/rlp/untrusted_rlp.rs +++ b/src/rlp/untrusted_rlp.rs @@ -436,6 +436,6 @@ fn test_rlp_display() { use rustc_serialize::hex::FromHex; let data = "f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470".from_hex().unwrap(); let rlp = UntrustedRlp::new(&data); - println!("{}", rlp); + assert_eq!(format!("{}", rlp), "[\"0x05\", \"0x010efbef67941f79b2\", \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\", \"0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470\"]"); } From 69f1f310b6f337ae37a70c3d93e77abc5ff76571 Mon Sep 17 00:00:00 2001 From: arkpar Date: Sat, 16 Jan 2016 13:29:51 +0100 Subject: [PATCH 372/379] Minor timers fix --- src/network/host.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index 8b1038c8a..99e8b5e7c 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -600,14 +600,12 @@ impl IoHandler> for Host where Messa FIRST_CONNECTION ... LAST_CONNECTION => self.connection_timeout(token, io), NODETABLE_DISCOVERY => {}, NODETABLE_MAINTAIN => {}, - _ => { - let protocol = *self.timers.get_mut(&token).expect("Unknown user timer token"); - match self.handlers.get_mut(protocol) { - None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, - Some(h) => { - h.timeout(&mut NetworkContext::new(io, protocol, Some(token), &mut self.connections, &mut self.timers), token); - } - } + _ => match self.timers.get_mut(&token).map(|p| *p) { + Some(protocol) => match self.handlers.get_mut(protocol) { + None => { warn!(target: "net", "No handler found for protocol: {:?}", protocol) }, + Some(h) => { h.timeout(&mut NetworkContext::new(io, protocol, Some(token), &mut self.connections, &mut self.timers), token); } + }, + None => {} // time not registerd through us } } } From a79644d982cf5ee1b7a6b0f7c5ea7ef5c4c1a987 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 16 Jan 2016 14:29:36 +0100 Subject: [PATCH 373/379] Fast SHA3. --- Cargo.toml | 4 + benches/trie.rs | 4 +- build.rs | 10 +++ src/network/host.rs | 10 +-- src/network/service.rs | 1 + src/sha3.rs | 19 +++-- src/tinykeccak.c | 178 +++++++++++++++++++++++++++++++++++++++++ src/uint.rs | 6 +- 8 files changed, 214 insertions(+), 18 deletions(-) create mode 100644 build.rs create mode 100644 src/tinykeccak.c diff --git a/Cargo.toml b/Cargo.toml index 4ed14551f..6b60fddf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,10 @@ license = "GPL-3.0" name = "ethcore-util" version = "0.1.0" authors = ["Ethcore "] +build = "build.rs" + +[build-dependencies] +gcc = "0.3" [dependencies] log = "0.3" diff --git a/benches/trie.rs b/benches/trie.rs index 8f1a2a42d..5f02b15cf 100644 --- a/benches/trie.rs +++ b/benches/trie.rs @@ -188,10 +188,10 @@ fn triehash_insertions_six_low(b: &mut Bencher) { } #[bench] -fn sha3x1000(b: &mut Bencher) { +fn sha3x10000(b: &mut Bencher) { b.iter(||{ let mut seed = H256::new(); - for _ in 0..1000 { + for _ in 0..10000 { seed = seed.sha3() } }) diff --git a/build.rs b/build.rs new file mode 100644 index 000000000..706da0907 --- /dev/null +++ b/build.rs @@ -0,0 +1,10 @@ + // build.rs + +// Bring in a dependency on an externally maintained `gcc` package which manages +// invoking the C compiler. +extern crate gcc; + +fn main() { + gcc::compile_library("libtinykeccak.a", &["src/tinykeccak.c"]); +} + diff --git a/src/network/host.rs b/src/network/host.rs index 8b1038c8a..4d8f0d056 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -561,13 +561,13 @@ impl IoHandler> for Host where Messa let port = self.info.config.listen_address.port(); self.info.listen_port = port; - //self.add_node("enode://c022e7a27affdd1632f2e67dffeb87f02bf506344bb142e08d12b28e7e5c6e5dbb8183a46a77bff3631b51c12e8cf15199f797feafdc8834aaf078ad1a2bcfa0@127.0.0.1:30303"); + self.add_node("enode://a9a921de2ff09a9a4d38b623c67b2d6b477a8e654ae95d874750cbbcb31b33296496a7b4421934e2629269e180823e52c15c2b19fc59592ec51ffe4f2de76ed7@127.0.0.1:30303"); // GO bootnodes - self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); // IE - self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR - self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG +// self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); // IE +// self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR +// self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG // ETH/DEV cpp-ethereum (poc-9.ethdev.com) - self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); +// self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); } fn stream_hup<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { diff --git a/src/network/service.rs b/src/network/service.rs index 48cc11152..4c333b8af 100644 --- a/src/network/service.rs +++ b/src/network/service.rs @@ -17,6 +17,7 @@ impl NetworkService where Message: Send + 'static { let mut io_service = try!(IoService::>::start()); let host = Box::new(Host::new()); let host_info = host.info.client_version.clone(); + info!("NetworkService::start(): id={:?}", host.info.id()); try!(io_service.register_handler(host)); Ok(NetworkService { io_service: io_service, diff --git a/src/sha3.rs b/src/sha3.rs index c23dfc7a9..ed3fe65a3 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,12 +1,15 @@ //! Wrapper around tiny-keccak crate. use std::mem::uninitialized; -use tiny_keccak::Keccak; -use bytes::{BytesConvertable,Populatable}; +use bytes::BytesConvertable; use hash::{H256, FixedHash}; pub const SHA3_EMPTY: H256 = H256( [0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70] ); +extern { + fn sha3_256(out: *mut u8, outlen: usize, input: *const u8, inputlen: usize) -> i32; +} + /// Types implementing this trait are sha3able. /// /// ``` @@ -29,17 +32,17 @@ pub trait Hashable { impl Hashable for T where T: BytesConvertable { fn sha3(&self) -> H256 { unsafe { - let mut keccak = Keccak::new_keccak256(); - keccak.update(self.bytes()); + let input: &[u8] = self.bytes(); let mut ret: H256 = uninitialized(); - keccak.finalize(ret.as_slice_mut()); + sha3_256(ret.as_mut_ptr(), ret.len(), input.as_ptr(), input.len()); ret } } fn sha3_into(&self, dest: &mut [u8]) { - let mut keccak = Keccak::new_keccak256(); - keccak.update(self.bytes()); - keccak.finalize(dest); + unsafe { + let input: &[u8] = self.bytes(); + sha3_256(dest.as_mut_ptr(), dest.len(), input.as_ptr(), input.len()); + } } } diff --git a/src/tinykeccak.c b/src/tinykeccak.c new file mode 100644 index 000000000..ddc3f5be2 --- /dev/null +++ b/src/tinykeccak.c @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +/** libkeccak-tiny + * + * A single-file implementation of SHA-3 and SHAKE. + * + * Implementor: David Leon Gil + * License: CC0, attribution kindly requested. Blame taken too, + * but not liability. + */ + +#define decshake(bits) \ + int shake##bits(uint8_t*, size_t, const uint8_t*, size_t); + +#define decsha3(bits) \ + int sha3_##bits(uint8_t*, size_t, const uint8_t*, size_t); + +decshake(128) +decshake(256) +decsha3(224) +decsha3(256) +decsha3(384) +decsha3(512) + +/******** The Keccak-f[1600] permutation ********/ + +/*** Constants. ***/ +static const uint8_t rho[24] = \ + { 1, 3, 6, 10, 15, 21, + 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, + 62, 18, 39, 61, 20, 44}; +static const uint8_t pi[24] = \ + {10, 7, 11, 17, 18, 3, + 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, + 20, 14, 22, 9, 6, 1}; +static const uint64_t RC[24] = \ + {1ULL, 0x8082ULL, 0x800000000000808aULL, 0x8000000080008000ULL, + 0x808bULL, 0x80000001ULL, 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x8aULL, 0x88ULL, 0x80008009ULL, 0x8000000aULL, + 0x8000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, 0x800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, 0x80000001ULL, 0x8000000080008008ULL}; + +/*** Helper macros to unroll the permutation. ***/ +#define rol(x, s) (((x) << s) | ((x) >> (64 - s))) +#define REPEAT6(e) e e e e e e +#define REPEAT24(e) REPEAT6(e e e e) +#define REPEAT5(e) e e e e e +#define FOR5(v, s, e) \ + v = 0; \ + REPEAT5(e; v += s;) + +/*** Keccak-f[1600] ***/ +static inline void keccakf(void* state) { + uint64_t* a = (uint64_t*)state; + uint64_t b[5] = {0}; + uint64_t t = 0; + uint8_t x, y; + + for (int i = 0; i < 24; i++) { + // Theta + FOR5(x, 1, + b[x] = 0; + FOR5(y, 5, + b[x] ^= a[x + y]; )) + FOR5(x, 1, + FOR5(y, 5, + a[y + x] ^= b[(x + 4) % 5] ^ rol(b[(x + 1) % 5], 1); )) + // Rho and pi + t = a[1]; + x = 0; + REPEAT24(b[0] = a[pi[x]]; + a[pi[x]] = rol(t, rho[x]); + t = b[0]; + x++; ) + // Chi + FOR5(y, + 5, + FOR5(x, 1, + b[x] = a[y + x];) + FOR5(x, 1, + a[y + x] = b[x] ^ ((~b[(x + 1) % 5]) & b[(x + 2) % 5]); )) + // Iota + a[0] ^= RC[i]; + } +} + +/******** The FIPS202-defined functions. ********/ + +/*** Some helper macros. ***/ + +#define _(S) do { S } while (0) +#define FOR(i, ST, L, S) \ + _(for (size_t i = 0; i < L; i += ST) { S; }) +#define mkapply_ds(NAME, S) \ + static inline void NAME(uint8_t* dst, \ + const uint8_t* src, \ + size_t len) { \ + FOR(i, 1, len, S); \ + } +#define mkapply_sd(NAME, S) \ + static inline void NAME(const uint8_t* src, \ + uint8_t* dst, \ + size_t len) { \ + FOR(i, 1, len, S); \ + } + +mkapply_ds(xorin, dst[i] ^= src[i]) // xorin +mkapply_sd(setout, dst[i] = src[i]) // setout + +#define P keccakf +#define Plen 200 + +// Fold P*F over the full blocks of an input. +#define foldP(I, L, F) \ + while (L >= rate) { \ + F(a, I, rate); \ + P(a); \ + I += rate; \ + L -= rate; \ + } + +/** The sponge-based hash construction. **/ +static inline int hash(uint8_t* out, size_t outlen, + const uint8_t* in, size_t inlen, + size_t rate, uint8_t delim) { + if ((out == NULL) || ((in == NULL) && inlen != 0) || (rate >= Plen)) { + return -1; + } + uint8_t a[Plen] = {0}; + // Absorb input. + foldP(in, inlen, xorin); + // Xor in the DS and pad frame. + a[inlen] ^= delim; + a[rate - 1] ^= 0x80; + // Xor in the last block. + xorin(a, in, inlen); + // Apply P + P(a); + // Squeeze output. + foldP(out, outlen, setout); + setout(a, out, outlen); + memset(a, 0, 200); + return 0; +} + +/*** Helper macros to define SHA3 and SHAKE instances. ***/ +#define defshake(bits) \ + int shake##bits(uint8_t* out, size_t outlen, \ + const uint8_t* in, size_t inlen) { \ + return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x1f); \ + } +#define defsha3(bits) \ + int sha3_##bits(uint8_t* out, size_t outlen, \ + const uint8_t* in, size_t inlen) { \ + if (outlen > (bits/8)) { \ + return -1; \ + } \ + return hash(out, outlen, in, inlen, 200 - (bits / 4), 0x01); \ + } + +/*** FIPS202 SHAKE VOFs ***/ +defshake(128) +defshake(256) + +/*** FIPS202 SHA3 FOFs ***/ +defsha3(224) +defsha3(256) +defsha3(384) +defsha3(512) + + + diff --git a/src/uint.rs b/src/uint.rs index 2a18a2fb0..58c25243d 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -600,10 +600,10 @@ macro_rules! construct_uint { let mut latch = false; for ch in data.iter().rev() { for x in 0..16 { - let ch = ch & (15u64 << ((15 - x) * 4) as u64) >> ((15 - x) * 4) as u64; - if !latch { latch = ch != 0 } + let nibble = (ch & (15u64 << ((15 - x) * 4) as u64)) >> (((15 - x) * 4) as u64); + if !latch { latch = nibble != 0 } if latch { - try!(write!(f, "{:01x}", ch)); + try!(write!(f, "{:x}", nibble)); } } } From 8e9ea7969aeca5b113f46749a80286a7fd98baa5 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 16 Jan 2016 15:29:36 +0100 Subject: [PATCH 374/379] Cleanup and docs. --- src/sha3.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sha3.rs b/src/sha3.rs index ed3fe65a3..c251edcfb 100644 --- a/src/sha3.rs +++ b/src/sha3.rs @@ -1,7 +1,7 @@ //! Wrapper around tiny-keccak crate. use std::mem::uninitialized; -use bytes::BytesConvertable; +use bytes::{BytesConvertable, Populatable}; use hash::{H256, FixedHash}; pub const SHA3_EMPTY: H256 = H256( [0xc5, 0xd2, 0x46, 0x01, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x03, 0xc0, 0xe5, 0x00, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x04, 0x5d, 0x85, 0xa4, 0x70] ); @@ -23,7 +23,10 @@ extern { /// } /// ``` pub trait Hashable { + /// Calculate SHA3 of this object. fn sha3(&self) -> H256; + + /// Calculate SHA3 of this object and place result into dest. fn sha3_into(&self, dest: &mut [u8]) { self.sha3().copy_to(dest); } @@ -32,9 +35,8 @@ pub trait Hashable { impl Hashable for T where T: BytesConvertable { fn sha3(&self) -> H256 { unsafe { - let input: &[u8] = self.bytes(); let mut ret: H256 = uninitialized(); - sha3_256(ret.as_mut_ptr(), ret.len(), input.as_ptr(), input.len()); + self.sha3_into(ret.as_slice_mut()); ret } } From 666cbe7d4793adc3c46bbca201163dd4ec33106e Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sat, 16 Jan 2016 15:45:33 +0100 Subject: [PATCH 375/379] Implementing pow, overflowing_pow only for uint. Adding missing docs. --- src/uint.rs | 59 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/uint.rs b/src/uint.rs index 546965ca4..f0df9681a 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -58,6 +58,7 @@ macro_rules! panic_on_overflow { } } } + pub trait Uint: Sized + Default + FromStr + From + FromJson + fmt::Debug + fmt::Display + PartialOrd + Ord + PartialEq + Eq + Hash { /// Size of this type. @@ -83,11 +84,19 @@ pub trait Uint: Sized + Default + FromStr + From + FromJson + fmt::Debug + /// Return the least number of bits needed to represent the number fn bits(&self) -> usize; + /// Return if specific bit is set fn bit(&self, index: usize) -> bool; + /// Return single byte fn byte(&self, index: usize) -> u8; + /// Get this Uint as slice of bytes fn to_bytes(&self, bytes: &mut[u8]); + /// Create `Uint(10**n)` fn exp10(n: usize) -> Self; + /// Return eponentation `self**other`. Panic on overflow. + fn pow(self, other: Self) -> Self; + /// Return wrapped eponentation `self**other` and flag if there was an overflow + fn overflowing_pow(self, other: Self) -> (Self, bool); } macro_rules! construct_uint { @@ -112,14 +121,12 @@ macro_rules! construct_uint { )) } - /// Conversion to u32 #[inline] fn low_u32(&self) -> u32 { let &$name(ref arr) = self; arr[0] as u32 } - /// Conversion to u64 #[inline] fn low_u64(&self) -> u64 { let &$name(ref arr) = self; @@ -147,6 +154,7 @@ macro_rules! construct_uint { } arr[0] } + /// Return the least number of bits needed to represent the number #[inline] fn bits(&self) -> usize { @@ -180,34 +188,33 @@ macro_rules! construct_uint { } #[inline] - fn exp10(n: usize) -> $name { + fn exp10(n: usize) -> Self { match n { - 0 => $name::from(1u64), - _ => $name::exp10(n - 1) * $name::from(10u64) + 0 => Self::from(1u64), + _ => Self::exp10(n - 1) * Self::from(10u64) } } #[inline] - fn zero() -> $name { + fn zero() -> Self { From::from(0u64) } #[inline] - fn one() -> $name { + fn one() -> Self { From::from(1u64) } - } - impl $name { - - pub fn pow(self, expon: $name) -> $name { - if expon == $name::zero() { - return $name::one() + /// Fast exponentation by squaring + /// https://en.wikipedia.org/wiki/Exponentiation_by_squaring + fn pow(self, expon: Self) -> Self { + if expon == Self::zero() { + return Self::one() } - let is_even = |x : &$name| x.low_u64() & 1 == 0; + let is_even = |x : &Self| x.low_u64() & 1 == 0; - let u_one = $name::one(); - let u_two = $name::from(2); + let u_one = Self::one(); + let u_two = Self::from(2); let mut y = u_one; let mut n = expon; let mut x = self; @@ -224,14 +231,16 @@ macro_rules! construct_uint { x * y } - pub fn overflowing_pow(self, expon: $name) -> ($name, bool) { - if expon == $name::zero() { - return ($name::one(), false) + /// Fast exponentation by squaring + /// https://en.wikipedia.org/wiki/Exponentiation_by_squaring + fn overflowing_pow(self, expon: Self) -> (Self, bool) { + if expon == Self::zero() { + return (Self::one(), false) } - let is_even = |x : &$name| x.low_u64() & 1 == 0; + let is_even = |x : &Self| x.low_u64() & 1 == 0; - let u_one = $name::one(); - let u_two = $name::from(2); + let u_one = Self::one(); + let u_two = Self::from(2); let mut y = u_one; let mut n = expon; let mut x = self; @@ -250,9 +259,11 @@ macro_rules! construct_uint { let res = overflowing!(x.overflowing_mul(y), overflow); (res, overflow) } + } + impl $name { /// Multiplication by u32 - fn mul_u32(self, other: u32) -> $name { + fn mul_u32(self, other: u32) -> Self { let $name(ref arr) = self; let mut carry = [0u64; $n_words]; let mut ret = [0u64; $n_words]; @@ -273,7 +284,7 @@ macro_rules! construct_uint { } /// Overflowing multiplication by u32 - fn overflowing_mul_u32(self, other: u32) -> ($name, bool) { + fn overflowing_mul_u32(self, other: u32) -> (Self, bool) { let $name(ref arr) = self; let mut carry = [0u64; $n_words]; let mut ret = [0u64; $n_words]; From 60678a21a658d699c4663733d80a155ec95e2b9b Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 16 Jan 2016 15:45:45 +0100 Subject: [PATCH 376/379] Put boot nodes back. --- src/network/host.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/network/host.rs b/src/network/host.rs index 8bafaefd0..32ac713ef 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -561,13 +561,13 @@ impl IoHandler> for Host where Messa let port = self.info.config.listen_address.port(); self.info.listen_port = port; - self.add_node("enode://a9a921de2ff09a9a4d38b623c67b2d6b477a8e654ae95d874750cbbcb31b33296496a7b4421934e2629269e180823e52c15c2b19fc59592ec51ffe4f2de76ed7@127.0.0.1:30303"); +// self.add_node("enode://a9a921de2ff09a9a4d38b623c67b2d6b477a8e654ae95d874750cbbcb31b33296496a7b4421934e2629269e180823e52c15c2b19fc59592ec51ffe4f2de76ed7@127.0.0.1:30303"); // GO bootnodes -// self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); // IE -// self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR -// self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG + self.add_node("enode://a979fb575495b8d6db44f750317d0f4622bf4c2aa3365d6af7c284339968eef29b69ad0dce72a4d8db5ebb4968de0e3bec910127f134779fbcb0cb6d3331163c@52.16.188.185:30303"); // IE + self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR + self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG // ETH/DEV cpp-ethereum (poc-9.ethdev.com) -// self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); + self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); } fn stream_hup<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { From 512eee04cf53c1863ccb560e96e80d28c391475c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 16 Jan 2016 16:26:46 +0100 Subject: [PATCH 377/379] Assign-operators for Uint. --- src/lib.rs | 1 + src/network/host.rs | 2 +- src/tinykeccak.c | 2 -- src/uint.rs | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1a7d873fe..4bc47e61c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![feature(op_assign_traits)] +#![feature(augmented_assignments)] #![feature(associated_consts)] #![feature(wrapping)] //! Ethcore-util library diff --git a/src/network/host.rs b/src/network/host.rs index 32ac713ef..90cbfdaad 100644 --- a/src/network/host.rs +++ b/src/network/host.rs @@ -567,7 +567,7 @@ impl IoHandler> for Host where Messa self.add_node("enode://de471bccee3d042261d52e9bff31458daecc406142b401d4cd848f677479f73104b9fdeb090af9583d3391b7f10cb2ba9e26865dd5fca4fcdc0fb1e3b723c786@54.94.239.50:30303"); // BR self.add_node("enode://1118980bf48b0a3640bdba04e0fe78b1add18e1cd99bf22d53daac1fd9972ad650df52176e7c7d89d1114cfef2bc23a2959aa54998a46afcf7d91809f0855082@52.74.57.123:30303"); // SG // ETH/DEV cpp-ethereum (poc-9.ethdev.com) - self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); + self.add_node("enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303"); } fn stream_hup<'s>(&'s mut self, io: &mut IoContext<'s, NetworkIoMessage>, stream: StreamToken) { diff --git a/src/tinykeccak.c b/src/tinykeccak.c index ddc3f5be2..466838913 100644 --- a/src/tinykeccak.c +++ b/src/tinykeccak.c @@ -1,6 +1,4 @@ #include -#include -#include #include /** libkeccak-tiny diff --git a/src/uint.rs b/src/uint.rs index 5073747eb..89834a85a 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -474,6 +474,38 @@ macro_rules! construct_uint { } } + // TODO: optimise and traitify. + + impl<'a> AddAssign<&'a $name> for $name { + fn add_assign(&mut self, other: &'a Self) { + *self = self.add(*other); + } + } + + impl<'a> SubAssign<&'a $name> for $name { + fn sub_assign(&mut self, other: &'a Self) { + *self = self.sub(*other); + } + } + + impl<'a> MulAssign<&'a $name> for $name { + fn mul_assign(&mut self, other: &'a Self) { + *self = self.mul(*other); + } + } + + impl<'a> DivAssign<&'a $name> for $name { + fn div_assign(&mut self, other: &'a Self) { + *self = self.div(*other); + } + } + + impl<'a> RemAssign<&'a $name> for $name { + fn rem_assign(&mut self, other: &'a Self) { + *self = self.rem(*other); + } + } + impl BitAnd<$name> for $name { type Output = $name; From da4f31b430d627584496087c28ca87542996bff6 Mon Sep 17 00:00:00 2001 From: Tomusdrw Date: Sat, 16 Jan 2016 16:44:47 +0100 Subject: [PATCH 378/379] Fixing tinykeccak to compile with c99 mode --- src/rlp/rlpin.rs | 2 +- src/tinykeccak.c | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rlp/rlpin.rs b/src/rlp/rlpin.rs index 371e953c9..159130750 100644 --- a/src/rlp/rlpin.rs +++ b/src/rlp/rlpin.rs @@ -147,5 +147,5 @@ fn break_it() { let h: Bytes = FromHex::from_hex("f84d0589010efbef67941f79b2a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").unwrap(); let r: Rlp = Rlp::new(&h); let u: U256 = r.val_at(1); - assert_eq!(format!("{}", u), "0x10efbef67941f79b2"); + assert_eq!(format!("{}", u), "19526463837540678066"); } diff --git a/src/tinykeccak.c b/src/tinykeccak.c index ddc3f5be2..c65b6cc71 100644 --- a/src/tinykeccak.c +++ b/src/tinykeccak.c @@ -61,8 +61,9 @@ static inline void keccakf(void* state) { uint64_t b[5] = {0}; uint64_t t = 0; uint8_t x, y; + int i; - for (int i = 0; i < 24; i++) { + for (i = 0; i < 24; i++) { // Theta FOR5(x, 1, b[x] = 0; @@ -96,7 +97,7 @@ static inline void keccakf(void* state) { #define _(S) do { S } while (0) #define FOR(i, ST, L, S) \ - _(for (size_t i = 0; i < L; i += ST) { S; }) + _({size_t i; for (i = 0; i < L; i += ST) { S; }}) #define mkapply_ds(NAME, S) \ static inline void NAME(uint8_t* dst, \ const uint8_t* src, \ From 0c2869d5420c3f1273d5203633bad931371cbaa2 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 16 Jan 2016 16:52:59 +0100 Subject: [PATCH 379/379] Tesys for U256 assign ops. --- src/uint.rs | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/src/uint.rs b/src/uint.rs index 89834a85a..00865387d 100644 --- a/src/uint.rs +++ b/src/uint.rs @@ -506,6 +506,36 @@ macro_rules! construct_uint { } } + impl AddAssign<$name> for $name { + fn add_assign(&mut self, other: Self) { + *self = self.add(other); + } + } + + impl SubAssign<$name> for $name { + fn sub_assign(&mut self, other: Self) { + *self = self.sub(other); + } + } + + impl MulAssign<$name> for $name { + fn mul_assign(&mut self, other: Self) { + *self = self.mul(other); + } + } + + impl DivAssign<$name> for $name { + fn div_assign(&mut self, other: Self) { + *self = self.div(other); + } + } + + impl RemAssign<$name> for $name { + fn rem_assign(&mut self, other: Self) { + *self = self.rem(other); + } + } + impl BitAnd<$name> for $name { type Output = $name; @@ -776,6 +806,37 @@ mod tests { use std::str::FromStr; use std::num::wrapping::OverflowingOps; + #[test] + pub fn assign_ops() { + let x: U256 = x!(69); + let y: U256 = x!(42); + { + let mut z = x; + z += y; + assert_eq!(z, x + y); + } + { + let mut z = x; + z -= y; + assert_eq!(z, x - y); + } + { + let mut z = x; + z *= y; + assert_eq!(z, x * y); + } + { + let mut z = x; + z /= y; + assert_eq!(z, x / y); + } + { + let mut z = x; + z %= y; + assert_eq!(z, x % y); + } + } + #[test] pub fn uint256_from() { let e = U256([10, 0, 0, 0]);