From d7c06c75c0aba62ffbb17bc13afefb786b80c8ea Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 22 Sep 2023 12:10:52 +1000 Subject: [PATCH 01/20] cleaning up error reporting --- libsodium | 2 +- src/ILog.h | 3 ++- src/unit_test.cpp | 14 ++++---------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/libsodium b/libsodium index 914033d..b6e1e94 160000 --- a/libsodium +++ b/libsodium @@ -1 +1 @@ -Subproject commit 914033d2c70831b86e94a25f27e61a94bc7d45c8 +Subproject commit b6e1e94cfb3d8dffdb1bb92b6102e5fad5fe6093 diff --git a/src/ILog.h b/src/ILog.h index 68f11ca..01bdc7d 100644 --- a/src/ILog.h +++ b/src/ILog.h @@ -19,11 +19,12 @@ private: public: virtual ~MyException() override = default; MyException() = delete; - explicit MyException(const std::string &m) noexcept :err(m.c_str()),err_number(-1){} explicit MyException(const char* sz) noexcept :err(sz),err_number(-1) {} + explicit MyException(const std::string& m) noexcept :err(m.c_str()), err_number(-1) {} explicit MyException(const char* sz, int i) noexcept :err(sz), err_number(i) {} explicit MyException(const char* sz, int i, const char*, const char*) noexcept; // usage throw MyException("error", __LINE__, __func__, __FILE__); + explicit MyException(const std::string& m, int i, const char* func, const char* file) noexcept : MyException(m.c_str(), i, func, file) {} explicit MyException(int, sqlite3*) noexcept; virtual const char* what() const override { return err.c_str(); diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 9ce7780..327c417 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -270,28 +270,22 @@ static bool CheckForUtfEnvironment(void) { wxString utfError{ wxT("") }; if constexpr (b_WINDOWS) { auto ACP{ GetACP() }; + // Check that windows thinks this is UTF8 utfEnvironment = utfEnvironment && (ACP == 65001); if (!utfEnvironment) { utfError += wxString::Format(wxT("current code page %d—should be 65001☹, "), ACP); } } auto FontEncoding{ wxLocale::GetSystemEncoding() }; - utfEnvironment = utfEnvironment && (false - || (FontEncoding == wxFONTENCODING_UTF8) - || (FontEncoding == wxFONTENCODING_UTF16BE) - || (FontEncoding == wxFONTENCODING_UTF16LE) - || (FontEncoding == wxFONTENCODING_UTF32BE) - || (FontEncoding == wxFONTENCODING_UTF32LE) - || (FontEncoding == wxFONTENCODING_SYSTEM) - ); + // check that wxWidgets thinks this is UTF8 + utfEnvironment = utfEnvironment && (FontEncoding == wxFONTENCODING_UTF8); if (!utfEnvironment) { utfError = wxString::Format(wxT("%swxFontEncoding %d—should be %d☹"), utfError, FontEncoding, wxFONTENCODING_UTF8); - wxLogMessage(wxT("%s"), utfError); + throw MyException(utfError.c_str(), __LINE__, __func__, SrcFilename); } - if (!utfEnvironment) { throw MyException(utfError); } } catch (const MyException& e) { errorCode = e.what_num(); From 3dabb9992c5382cd8e55655cd69c83f3e2f9588c Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 22 Sep 2023 18:36:07 +1000 Subject: [PATCH 02/20] updating to version of wxWidgets that tells windows to be unicode --- wxWidgets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wxWidgets b/wxWidgets index 24f3e55..8ef067c 160000 --- a/wxWidgets +++ b/wxWidgets @@ -1 +1 @@ -Subproject commit 24f3e551d4cf583573eeb838d04a3c7c3b3a6121 +Subproject commit 8ef067c7317b302068f1b774c92c5250052c2a53 From 9a12dbbd7e53da61a0b5d1b71008d093388eef0c Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 22 Sep 2023 19:49:30 +1000 Subject: [PATCH 03/20] minimally broken branch to isolate what is broken --- src/ristretto255.h | 55 ++++++++++------------------------------------ 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/src/ristretto255.h b/src/ristretto255.h index 1a8c31a..054b511 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -449,35 +449,36 @@ namespace ristretto255 { template ristretto255::hsh& operator << (const T& j) { + int i{ 1 }; if constexpr (std::is_same_v, std::span >) { - int i = crypto_generichash_blake2b_update( + i = crypto_generichash_blake2b_update( &(this->st), &j[0], j.size() ); - if (i) throw HashReuseException(); - return *this; } else if constexpr (std::is_same_v, const char*>) { - int i = crypto_generichash_blake2b_update( + i = crypto_generichash_blake2b_update( &(this->st), (const unsigned char*)(j), strlen(j) + 1 ); - if (i) throw HashReuseException(); - return *this; } - else { + else if constexpr (is_serializable::value) { auto sj = ro::serialize(j); - int i = crypto_generichash_blake2b_update( + i = crypto_generichash_blake2b_update( &(this->st), (const unsigned char*)&sj[0], sj.size() ); - if (i) throw HashReuseException(); - return *this; } + else { + static_assert(false, "don't know a machine and compiler independent representation of this type"); + } + if (i) throw HashReuseException(); + return *this; } + template::value, int >::type dummy_arg = 0 >explicit hsh(const T first, Args... args) { @@ -502,40 +503,6 @@ namespace ristretto255 { } } }; - /* - template - ristretto255::hsh& operator <<(ristretto255::hsh &u, const T& j) { - if constexpr (std::is_same_v, std::span >) { - int i = crypto_generichash_blake2b_update( - &u.st, - &j[0], - j.size() - ); - if (i) throw HashReuseException(); - return u; - } - else if constexpr (std::is_same_v, const char*>) { - int i = crypto_generichash_blake2b_update( - &u.st, - (const unsigned char *)(j), - strlen(j) + 1 - ); - if (i) throw HashReuseException(); - return u; - } - else { - auto sj = ro::serialize(j); - int i = crypto_generichash_blake2b_update( - &u.st, - (const unsigned char*)&sj[0], - sj.size() - ); - if (i) throw HashReuseException(); - return u; - } - } - */ - // This constructs a finalized hash. // If it has one argument, and that argument is hsh (unfinalized hash) object, From 1b0d5148e8e345f6eba68f91148adb0df8fdb30a Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 22 Sep 2023 20:35:28 +1000 Subject: [PATCH 04/20] putting in additional SFINAE guards in an effort to hunt down pesky and mysterious warnings --- src/ristretto255.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ristretto255.h b/src/ristretto255.h index 054b511..e2b350d 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -448,7 +448,10 @@ namespace ristretto255 { } template - ristretto255::hsh& operator << (const T& j) { + typename std::enable_if< + ro::is_serializable::value, + ristretto255::hsh& + >::type operator << (const T& j) { int i{ 1 }; if constexpr (std::is_same_v, std::span >) { i = crypto_generichash_blake2b_update( @@ -464,7 +467,8 @@ namespace ristretto255 { strlen(j) + 1 ); } - else if constexpr (is_serializable::value) { + else { + static_assert(is_serializable::value, "don't know a machine and compiler independent representation of this type"); auto sj = ro::serialize(j); i = crypto_generichash_blake2b_update( &(this->st), @@ -472,9 +476,6 @@ namespace ristretto255 { sj.size() ); } - else { - static_assert(false, "don't know a machine and compiler independent representation of this type"); - } if (i) throw HashReuseException(); return *this; } From 89ebcee05473e50a9d90fb00dc7955f63d34ddda Mon Sep 17 00:00:00 2001 From: Cheng Date: Sat, 23 Sep 2023 15:58:56 +1000 Subject: [PATCH 05/20] fixed those irritating uninitialized memory warnings by annotating parameters --- msvc/wallet.vcxproj | 2 ++ src/ristretto255.h | 26 ++++++++++++++------------ src/testbed.cpp | 31 ++++++++++++++++++++----------- src/unit_test.cpp | 4 ++-- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/msvc/wallet.vcxproj b/msvc/wallet.vcxproj index e9799ea..d9dda3a 100644 --- a/msvc/wallet.vcxproj +++ b/msvc/wallet.vcxproj @@ -71,6 +71,7 @@ false false ProgramDatabase + true Windows @@ -108,6 +109,7 @@ /Zc:__cplusplus /utf-8 %(AdditionalOptions) false false + true Windows diff --git a/src/ristretto255.h b/src/ristretto255.h index e2b350d..67269cf 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -431,7 +431,9 @@ namespace ristretto255 { // It always a wise precaution to not use old type C arays, or wrap them // in a span. // Old type zero terminated strings work. The trailing zero is included - // in the hash + // in the hash, thus hash("the quick ", "brown fox") != hash("the quick brown fox") + + template class hsh { public: static_assert(hashsize > 63 && hashsize % 64 == 0 && crypto_generichash_BYTES_MIN * 8 <= hashsize && hashsize <= crypto_generichash_BYTES_MAX * 8, "Bad hash size."); @@ -504,6 +506,7 @@ namespace ristretto255 { } } }; + static_assert(!ro::is_serializable >::value, "Don't want to partially hash partial hashes"); // This constructs a finalized hash. // If it has one argument, and that argument is hsh (unfinalized hash) object, @@ -550,7 +553,6 @@ namespace ristretto255 { assert(i == 0); if (i) throw HashReuseException(); } - static_assert(!ro::is_serializable >::value, "Don't want to partially hash partial hashes"); template::value, int >::type dummy_arg = 0 >explicit hash(const T& first, Args... args) { @@ -616,7 +618,7 @@ namespace ristretto255 { } point operator+(const point &pt) const& { point me; - auto i{ crypto_core_ristretto255_add(&me.blob[0], &blob[0], &pt.blob[0]) }; + auto i{ crypto_core_ristretto255_add(_Out_ & me.blob[0], &blob[0], &pt.blob[0])}; assert(i == 0); if (i != 0)throw NonRandomScalarException(); return me; @@ -624,7 +626,7 @@ namespace ristretto255 { point operator-(const point& pt) const& { point me; - auto i{ crypto_core_ristretto255_sub(&me.blob[0], &blob[0], &pt.blob[0]) }; + auto i{ crypto_core_ristretto255_sub(_Out_ &me.blob[0], &blob[0], &pt.blob[0]) }; assert(i == 0); if (i != 0)throw NonRandomScalarException(); return me; @@ -646,7 +648,7 @@ namespace ristretto255 { static point random(void) { point me; - crypto_core_ristretto255_random(&(me.blob[0])); + crypto_core_ristretto255_random(_Out_ &(me.blob[0])); return me; } @@ -699,14 +701,14 @@ namespace ristretto255 { // } scalar operator+(const scalar sclr) const& { scalar me; - crypto_core_ristretto255_scalar_add(&me.blob[0], &blob[0], &sclr.blob[0]); + crypto_core_ristretto255_scalar_add( _Out_&me.blob[0], &blob[0], &sclr.blob[0]); return me; } static_assert(sizeof(scalar::blob) == 32, "compiled"); scalar multiplicative_inverse() const &{ scalar me; - auto i = crypto_core_ristretto255_scalar_invert(&me.blob[0], &blob[0]); + auto i = crypto_core_ristretto255_scalar_invert(_Out_&me.blob[0], &blob[0]); assert(i == 0); if (i != 0)throw NonRandomScalarException(); return me; @@ -714,13 +716,13 @@ namespace ristretto255 { scalar operator-(const scalar& sclr) const& { scalar me; - crypto_core_ristretto255_scalar_sub(&me.blob[0], &blob[0], &sclr.blob[0]); + crypto_core_ristretto255_scalar_sub(_Out_&me.blob[0], &blob[0], &sclr.blob[0]); return me; } scalar operator*(const scalar& sclr) const& { scalar me; - crypto_core_ristretto255_scalar_mul(&me.blob[0], &blob[0], &sclr.blob[0]); + crypto_core_ristretto255_scalar_mul(_Out_&me.blob[0], &blob[0], &sclr.blob[0]); return me; } @@ -734,7 +736,7 @@ namespace ristretto255 { point operator*(const point& pt) const& { point me; - auto i{ crypto_scalarmult_ristretto255(&me.blob[0], &blob[0], &pt.blob[0]) }; + auto i{ crypto_scalarmult_ristretto255(_Out_&me.blob[0], &blob[0], &pt.blob[0]) }; assert(i == 0); if (i != 0)throw NonRandomScalarException(); return me; @@ -742,7 +744,7 @@ namespace ristretto255 { point timesBase() const& { point me; - auto i{ crypto_scalarmult_ristretto255_base(&me.blob[0], &blob[0]) }; + auto i{ crypto_scalarmult_ristretto255_base(_Out_ & me.blob[0], &blob[0]) }; assert(i == 0); if (i != 0)throw NonRandomScalarException(); return me; @@ -755,7 +757,7 @@ namespace ristretto255 { static scalar random(void) { scalar me; - crypto_core_ristretto255_scalar_random(&me.blob[0]); + crypto_core_ristretto255_scalar_random(_Out_ & me.blob[0]); return me; } diff --git a/src/testbed.cpp b/src/testbed.cpp index b76514e..93f47f7 100644 --- a/src/testbed.cpp +++ b/src/testbed.cpp @@ -28,6 +28,7 @@ namespace testbed { using ristretto255::hash, ristretto255::hsh, ristretto255::scalar, ristretto255::point, ro::serialize, ro::bin2hex, ro::hex2bin, ro::bin2hex, ro::fasthash,ro::CompileSizedString ; + static constexpr char SrcFilename[]{ "src/testbed.cpp" }; /* experimental code called during unit test Anything here is a residue of forgotten experiments, @@ -36,21 +37,29 @@ namespace testbed { This is a playground, where you can do stuff without worrying you might inadvertently break something that matters -No mechanism for input is available. You generally do not need it because you - hard code the testing data, and detect errors with asserts, rather than exceptions - but, of course, it can post a dialog using postmessage, then immediately return - and the dialog can then call anything. +Output goes to the unit test log. -Uncaught exceptions result in unit test failure, but not in an error - message in the main program UI. +No mechanism for input is available. You generally do not need it + because you hard code the testing data, and detect errors with + asserts, rather than exceptions but, of course, it can post a + dialog using postmessage, and the dialog can then call anything. -If using a dialog, exceptions within the dialog will result in an error message in the - main program UI, rather than in the unit test result, since the unit test - is over before the dialog runs. +Uncaught exceptions result in unit test failure, and an error message + at the end of the unit test, but not in an error message in the + main program UI. + +If using postmessage, execution of the code waits for the dialog to + return, data from the dialog can be used in the testbed code, and + uncaught exceptions in the dialog will result unit test failure + and be reported in the unit test log. + +If using queumessage, the testbed code will complete while the dialog + is waiting, data cannot be returned for use in the testbed code, + and uncaught exceptions will appear in the main UI. */ void testbed() { -// queue_error_message("hello world"); - ascii2test(); + // queue_error_message("hello world"); + // throw MyException("hello world exception", __LINE__, __func__, SrcFilename); } } diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 327c417..225a09e 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -63,7 +63,7 @@ static bool EndUnitTest() { unit_test_action = &noaction; modal_dialog_hook.Unregister(); next_action = &unexpected_unit_test; - static std::string intestbed("Testbed: "); + std::string intestbed("Testbed: "); try { testbed::testbed(); throw EndUnitTestOfExceptions(); @@ -71,7 +71,7 @@ static bool EndUnitTest() { catch (const EndUnitTestOfExceptions&) {} catch (const MyException& e) { errorCode = e.what_num(); - szError = e.what(); + szError = intestbed + e.what(); ILogError(szError.c_str()); } catch (const std::exception& e) { From fc9f82b6e510ff40b8ac397fe241247ae4cb1361 Mon Sep 17 00:00:00 2001 From: Cheng Date: Sun, 24 Sep 2023 15:21:18 +1000 Subject: [PATCH 06/20] Documented a bit of code that I had started to forget. Remade hashing according to the dry principle, eliminating much code repetition. Introduced c++20 "concepts" to radically reduce verbose and incomprehensible template metacode. modified: src/db_accessors.h modified: src/ristretto255.h modified: src/testbed.cpp --- src/ILog.cpp | 7 +++- src/db_accessors.h | 2 +- src/ristretto255.h | 97 +++++++++++++++++++--------------------------- src/testbed.cpp | 10 +++-- 4 files changed, 51 insertions(+), 65 deletions(-) diff --git a/src/ILog.cpp b/src/ILog.cpp index 1fa6f2c..f02728e 100644 --- a/src/ILog.cpp +++ b/src/ILog.cpp @@ -35,8 +35,11 @@ void queue_error_message(const char* psz) { void queue_fatal_error(const char* psz) { // Used where throwing immediately would be disastrous, as in a destructor or when constructing the main frame if (!errorCode)errorCode = 10; - queue_error_message(psz); - singletonFrame->Close(); + queue_error_message(psz); // Message continues to display, + // even after App window that contains the frame has closed + singletonFrame->Close(); // Generates close event, which is handled as usual after this returns + // Main window closes after the close message is handled, but App still running, displaying the message + // put up by queue_error_message. When that message is closed, App finally terminates. } MyException::MyException(const char* sz, int i, const char* func__, const char* FILE__) noexcept : diff --git a/src/db_accessors.h b/src/db_accessors.h index e68d6c0..531176e 100644 --- a/src/db_accessors.h +++ b/src/db_accessors.h @@ -107,7 +107,7 @@ namespace ro { template < typename T, typename std::enable_if::value, int >::type dummy_arg = 0 > void bind(int i, const T& j) { - static_assert(ro::is_serializable::value, "Don't know how to store this type in a database"); + static_assert(ro::serializable, "Don't know how to store this type in a database"); (*this)->Isqlite3_bind(i, ro::serialize(j)); } typedef Icompiled_sql::sql_result result; diff --git a/src/ristretto255.h b/src/ristretto255.h index 67269cf..883902b 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -175,6 +175,8 @@ namespace ro { apt to unhelpfully and unexpectedly turn it into a wxString, If you make wxStrings hashable, suprising things become hashable. + However, we do make the strange data structure provided by wxString.ToUTF8() hashable, + so that the wxString will not be implicitly hashable, but will be explicitly hashable. */ // data structure containing a serialized signed integer. @@ -334,7 +336,7 @@ namespace ro { currency is fixed at 2^64-1 which will suffice for a thousand years. Or we might allow arbitrary precision floating point with powers of a thousand, so that sensible numbers to a human are represented by - sensible numbers in the actuall representation. + sensible numbers in the actual representation. secret keys, scalars are actually much larger numbers, modulo 0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ecU @@ -359,8 +361,13 @@ namespace ro { } static constexpr bool value = is_serializable::template test(); }; - static_assert(ro::is_serializable>::value); - static_assert(ro::is_serializable>::value); + + template + concept serializable = is_serializable::value; + + static_assert( !serializable + && serializable, char*, std::span>, + "concepts needed"); template ro::CompileSizedString< (2 * sizeof(T))>bin2hex(const T& pt) { ro::CompileSizedString< (2 * sizeof(T))>sz; @@ -392,7 +399,12 @@ namespace ro { } //End ro namespace namespace ristretto255 { - using ro::to_base64_string, ro::is_serializable; + using + ro::to_base64_string, ro::is_serializable, + ro::serialize, ro::bin2hex, ro::hex2bin, + ro::bin2hex, ro::CompileSizedString, + ro::serializable; + ; // a class representing ristretto255 elliptic points class point; @@ -449,64 +461,35 @@ namespace ristretto255 { assert(i == 0); } - template - typename std::enable_if< - ro::is_serializable::value, - ristretto255::hsh& - >::type operator << (const T& j) { - int i{ 1 }; - if constexpr (std::is_same_v, std::span >) { - i = crypto_generichash_blake2b_update( - &(this->st), - &j[0], - j.size() - ); - } - else if constexpr (std::is_same_v, const char*>) { - i = crypto_generichash_blake2b_update( - &(this->st), - (const unsigned char*)(j), - strlen(j) + 1 - ); - } - else { - static_assert(is_serializable::value, "don't know a machine and compiler independent representation of this type"); - auto sj = ro::serialize(j); - i = crypto_generichash_blake2b_update( - &(this->st), - (const unsigned char*)&sj[0], - sj.size() - ); + template + hsh& hashinto(const T& j, Args... args) { + auto sj = ro::serialize(j); + int i = crypto_generichash_blake2b_update( + &(this->st), + (const unsigned char*)&sj[0], + sj.size() + ); + if (i) throw HashReuseException(); + if constexpr (sizeof...(args) > 0) { + (*this).hashinto(args...); } + return *this; + } + + template hsh& operator << (const T& j) { + auto sj = ro::serialize(j); + auto i = crypto_generichash_blake2b_update( + &(this->st), + (const unsigned char*)&sj[0], + sj.size() + ); if (i) throw HashReuseException(); return *this; } - template::value, int >::type dummy_arg = 0 - >explicit hsh(const T first, Args... args) { - int i{ crypto_generichash_blake2b_init( - &st, - nullptr,0, - hashsize / 8) - }; - assert(i == 0); - (*this) << first; - if constexpr (sizeof...(args) > 0) { - (*this).hashinto(args...); - } - } - template::value, int >::type dummy_arg = 0 - > void hashinto(const T first, Args... args) { - *this << first; - if constexpr (sizeof...(args) > 0) { - (*this).hashinto(args...); - } - } }; - static_assert(!ro::is_serializable >::value, "Don't want to partially hash partial hashes"); + static_assert(!serializable >, "Don't want to partially hash partial hashes"); // This constructs a finalized hash. // If it has one argument, and that argument is hsh (unfinalized hash) object, @@ -553,9 +536,7 @@ namespace ristretto255 { assert(i == 0); if (i) throw HashReuseException(); } - template::value, int >::type dummy_arg = 0 - >explicit hash(const T& first, Args... args) { + template< serializable T, typename... Args>explicit hash(const T& first, Args... args) { hsh in; in << first; if constexpr (sizeof...(args) > 0) { diff --git a/src/testbed.cpp b/src/testbed.cpp index 93f47f7..1a44eac 100644 --- a/src/testbed.cpp +++ b/src/testbed.cpp @@ -25,11 +25,13 @@ void ascii2test(); extern const uint8_t* const ascii2six; namespace testbed { - using ristretto255::hash, ristretto255::hsh, ristretto255::scalar, + using /*ristretto255::hash, ristretto255::hsh, */ristretto255::scalar, ristretto255::point, ro::serialize, ro::bin2hex, ro::hex2bin, - ro::bin2hex, ro::fasthash,ro::CompileSizedString ; + ro::bin2hex, ro::fasthash, ro::CompileSizedString, + ro::base58, ro::serializable; static constexpr char SrcFilename[]{ "src/testbed.cpp" }; + /* experimental code called during unit test Anything here is a residue of forgotten experiments, and can safely be thrown away @@ -54,8 +56,8 @@ If using postmessage, execution of the code waits for the dialog to and be reported in the unit test log. If using queumessage, the testbed code will complete while the dialog - is waiting, data cannot be returned for use in the testbed code, - and uncaught exceptions will appear in the main UI. + is waiting, data cannot be returned for use in the testbed code, + and uncaught exceptions in the dialog queued will appear in the main UI. */ void testbed() { From 883bd8ebe8a0e6660a65985074040b1e96052a3e Mon Sep 17 00:00:00 2001 From: Cheng Date: Sun, 24 Sep 2023 20:04:19 +1000 Subject: [PATCH 07/20] std::format seems like a great idea. Used it in one place. Better than printf derived functionality. But they never bothered to think through supporting user types, and that is a complete mess that is grossly inconsistent or simply not working from one compiler to the next. --- src/ILog.cpp | 5 ++--- src/db_accessors.h | 2 +- src/mpir_and_base58.h | 3 ++- src/ristretto255.h | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ILog.cpp b/src/ILog.cpp index f02728e..6ef3132 100644 --- a/src/ILog.cpp +++ b/src/ILog.cpp @@ -46,7 +46,6 @@ MyException::MyException(const char* sz, int i, const char* func__, const char* // usage // throw MyException("Expected wallet file not found", __LINE__, __func__, __FILE__); err_number(i) { - char buff[20]; - snprintf(buff, 20, "%d", i); - err = std::string(sz) + "\nline " + buff + ", function " + func__ + ", file " + FILE__; + err = std::format(R"|({} +line {}, function {}, file {})|", sz, i, func__, FILE__); } diff --git a/src/db_accessors.h b/src/db_accessors.h index 531176e..500c86f 100644 --- a/src/db_accessors.h +++ b/src/db_accessors.h @@ -36,7 +36,7 @@ namespace ro { sql(std::unique_ptr&& p) :std::unique_ptr(p.release()) { } ~sql() = default; template auto column(int i) const { - if constexpr (ro::is_blob_field_type::value) { + if constexpr (ro::blob_type) { auto st = (*this)->Isqlite3_column_blob(i); if (st.size_bytes() != sizeof(T)) throw BadDataException(); static_assert (std::is_standard_layout(), "not standard layout"); diff --git a/src/mpir_and_base58.h b/src/mpir_and_base58.h index f18a230..13f2389 100644 --- a/src/mpir_and_base58.h +++ b/src/mpir_and_base58.h @@ -57,7 +57,7 @@ namespace ro { bool is_alphanumeric_fixed_length(unsigned int, const char*); template typename std::enable_if< - ro::is_blob_field_type::value, + ro::blob_type, decltype(T::type_indentifier, uint32_t()) >::type fasthash(const T& p_blob) { static_assert(sizeof(T) % 8 == 0, "fasthash assumes a multiple of 8 bytes"); @@ -181,3 +181,4 @@ namespace ro { this->operator char* [this->length] = '\0'; } } + diff --git a/src/ristretto255.h b/src/ristretto255.h index 883902b..a868115 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -138,17 +138,16 @@ namespace ro { static constexpr bool value = is_blob_field_type::template test(); }; + template concept blob_type = ro::is_blob_field_type::value; + + // At present our serial classes consist of std::span and custom classes that publicly inherit from std::span // To handle compound objects, add custom classes inheriting from std::span[n] // template class that generates a std::span of bytes over the blob // field of any object containing a blob record, which is normally sufficient // for a machine independent representation of that object - template - std::enable_if_t< - is_blob_field_type::value, - std::span - > serialize(const T& pt) { + template std::span serialize(const T& pt) { return serialize(pt.blob); } @@ -567,7 +566,7 @@ namespace ristretto255 { { // We will be reading points from the database, as blobs, // reading them from the network as blobs, - // and reading them from human entered text as base52 encoded blobs. + // and reading them from human entered text as base58 encoded blobs. // Therefore, invalid point initialization data is all too possible. public: static constexpr unsigned int type_indentifier = 1; @@ -770,6 +769,7 @@ namespace ristretto255 { } }; + static_assert(ro::blob_type && !ro::blob_type); static_assert(ro::is_blob_field_type::value); static_assert(ro::is_blob_field_type::value); static_assert(ro::is_blob_field_type::value); From fefb99bb33f05e3b349158243f7ba8f70c203b03 Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 25 Sep 2023 10:23:11 +1000 Subject: [PATCH 08/20] Discovered that modifying the wxWidgets manifest has no effect Restored the wallet manifest. Discovered that local wxString variables are apt to get wiped during a throw. Dangerous to pass a wXstring.utf8() to an exception --- msvc/wallet.vcxproj | 10 +++++++--- src/ISqlit3Impl.cpp | 2 +- src/unit_test.cpp | 15 +++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/msvc/wallet.vcxproj b/msvc/wallet.vcxproj index d9dda3a..1a390cf 100644 --- a/msvc/wallet.vcxproj +++ b/msvc/wallet.vcxproj @@ -80,13 +80,14 @@ mpir.lib;mpirxx.lib;libsodium.lib;%(AdditionalDependencies) - wallet.manifest %(AdditionalManifestFiles) + + - false + true - PerMonitorHighDPIAware + false @@ -123,6 +124,9 @@ + + true + diff --git a/src/ISqlit3Impl.cpp b/src/ISqlit3Impl.cpp index 20ec69f..c42d8e0 100644 --- a/src/ISqlit3Impl.cpp +++ b/src/ISqlit3Impl.cpp @@ -25,7 +25,7 @@ void sqlite3_init() { errorCode = 7; szError = "Fatal Error: Sqlite library did not init."; // Cannot log the error, because logging not set up yet, so logging itself causes an exception - throw FatalException(szError.c_str()); + throw FatalException(szError); } } diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 225a09e..e4ec4e0 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -264,27 +264,26 @@ static bool checkDataConversionsProduceExpected(void){ } static bool CheckForUtfEnvironment(void) { - ILogMessage("\tChecking for UTF locale."); + ILogMessage("\tChecking for UTF locale. ☹"); try { bool utfEnvironment{ true }; - wxString utfError{ wxT("") }; if constexpr (b_WINDOWS) { auto ACP{ GetACP() }; // Check that windows thinks this is UTF8 utfEnvironment = utfEnvironment && (ACP == 65001); if (!utfEnvironment) { - utfError += wxString::Format(wxT("current code page %d—should be 65001☹, "), ACP); + szError = std::format("current code page {}, —should be 65001☹, ", ACP); } } auto FontEncoding{ wxLocale::GetSystemEncoding() }; // check that wxWidgets thinks this is UTF8 utfEnvironment = utfEnvironment && (FontEncoding == wxFONTENCODING_UTF8); if (!utfEnvironment) { - utfError = wxString::Format(wxT("%swxFontEncoding %d—should be %d☹"), - utfError, - FontEncoding, - wxFONTENCODING_UTF8); - throw MyException(utfError.c_str(), __LINE__, __func__, SrcFilename); + szError += std::format("swxFontEncoding {}, – should be {} ☹", + (int)FontEncoding, + (int)wxFONTENCODING_UTF8 + ); + throw MyException(szError, __LINE__, __func__, SrcFilename); } } catch (const MyException& e) { From 8e7225440aa8eb42f58b33ba4e98844f7f19d250 Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 25 Sep 2023 11:35:18 +1000 Subject: [PATCH 09/20] my exception code was too clever by half --- src/unit_test.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/unit_test.cpp b/src/unit_test.cpp index e4ec4e0..989106c 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -264,26 +264,27 @@ static bool checkDataConversionsProduceExpected(void){ } static bool CheckForUtfEnvironment(void) { - ILogMessage("\tChecking for UTF locale. ☹"); + ILogMessage("\tChecking for UTF locale. ☺"); try { bool utfEnvironment{ true }; + std::string cum{}; if constexpr (b_WINDOWS) { auto ACP{ GetACP() }; // Check that windows thinks this is UTF8 utfEnvironment = utfEnvironment && (ACP == 65001); if (!utfEnvironment) { - szError = std::format("current code page {}, —should be 65001☹, ", ACP); + cum += std::format("current code page {}, —should be 65001☹, ", ACP); } } auto FontEncoding{ wxLocale::GetSystemEncoding() }; // check that wxWidgets thinks this is UTF8 utfEnvironment = utfEnvironment && (FontEncoding == wxFONTENCODING_UTF8); if (!utfEnvironment) { - szError += std::format("swxFontEncoding {}, – should be {} ☹", + cum += std::format("swxFontEncoding {}, —should be {} ☹", (int)FontEncoding, (int)wxFONTENCODING_UTF8 ); - throw MyException(szError, __LINE__, __func__, SrcFilename); + throw MyException(cum, __LINE__, __func__, SrcFilename); } } catch (const MyException& e) { From f7876905e3a322aba5202949c32fd91088f8663b Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 25 Sep 2023 18:30:42 +1000 Subject: [PATCH 10/20] Simplifying the serialization to and from VLQ integer format with C++20 concepts. Moving serialization to a new header, serialization.h --- msvc/wallet.vcxproj | 1 + src/ristretto255.h | 341 -------------------------------------------- src/serialization.h | 340 +++++++++++++++++++++++++++++++++++++++++++ src/stdafx.h | 1 + 4 files changed, 342 insertions(+), 341 deletions(-) create mode 100644 src/serialization.h diff --git a/msvc/wallet.vcxproj b/msvc/wallet.vcxproj index 1a390cf..99865da 100644 --- a/msvc/wallet.vcxproj +++ b/msvc/wallet.vcxproj @@ -144,6 +144,7 @@ + diff --git a/src/ristretto255.h b/src/ristretto255.h index a868115..3a4eb57 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -54,348 +54,7 @@ void randombytes_buf(std::span in); void randombytes_buf(std::span in); -namespace ro { - // Decay to pointer is dangerously convenient, - // but in some situations it is just convenient - // This class provides an std:array one larger - // than the compile time string size, which decays - // to char*, std::string, and wxString - // In some code, this is ambiguous, so casts - // must sometimes be explicitly invoked. - template - class CompileSizedString : public std::array{ - public: - static constexpr int length{ stringlen }; - CompileSizedString() { - *(this->rbegin()) = '0'; - } - CompileSizedString(char *psz) { - auto tsz{ this->rbegin() }; - *tsz = '0'; - if (psz != nullptr) { - auto usz = tsz + strlen; - while (tsz < usz && *psz != '\0') - *tsz++ = *psz++; - *tsz = '\0'; - } - } - operator char* () & { - char* pc = &(static_cast*>(this)->operator[](0)); - return pc; - } - - operator const char* () const& { - const char* pc = &(static_cast*>(this)->operator[](0)); - return pc; - } - operator const char* () const&& { - const char* pc = &(static_cast*>(this)->operator[](0)); - return pc; - } - operator std::string() const& { - return std::string((const char*)*this, this->length); - } - operator std::string() const&& { - return std::string((const char*)*this, this->length); - } - operator wxString() const& { - return wxString::FromUTF8Unchecked((const char*)(*this)); - } - operator std::span() const& { - return std::span(static_cast((char*)*this), stringlen + 1); - } - operator wxString() const&& { - return wxString::FromUTF8Unchecked((const char*)(*this)); - } - operator std::span() const&& { - return std::span(static_cast((char*)*this), stringlen + 1); - } - }; - - // This template generates a span over an indexable byte type, - // such as a C array or an std::array, but not pointers - template < typename T> - std::enable_if_t< - !std::is_pointer::value && - sizeof(std::declval()[0]) == 1, - std::span - > serialize(const T& a) { - return std::span(static_cast(static_cast(&a[0])), std::size(a)); - } - - // Compile time test to see if a type has a blob array member - // This can be used in if constexpr (is_blob_field_type::value) - // By convention, blob fields are an std::array of unsigned bytes - // therefore already serializable. - template struct is_blob_field_type{ - template static constexpr decltype(std::declval().blob.size(), bool()) test() { - return sizeof(std::declval().blob[0])==1; - } - template static constexpr bool test(int = 0) { - return false; - } - static constexpr bool value = is_blob_field_type::template test(); - }; - - template concept blob_type = ro::is_blob_field_type::value; - - - // At present our serial classes consist of std::span and custom classes that publicly inherit from std::span - // To handle compound objects, add custom classes inheriting from std::span[n] - - // template class that generates a std::span of bytes over the blob - // field of any object containing a blob record, which is normally sufficient - // for a machine independent representation of that object - template std::span serialize(const T& pt) { - return serialize(pt.blob); - } - - // method that assumes that any char * pointer points a null terminated string - // and generates a std::span of bytes over the string including the terminating - // null. - // we assume the string is already machine independent, which is to say, we assume - // it comes from a utf8 locale. - - inline auto serialize(const char* sp) { return std::span(static_cast(static_cast(sp)), strlen(sp) + 1); } - - inline auto serialize(const decltype(std::declval().ToUTF8()) sz){ - return serialize(static_cast(sz)); - } - /* - inline auto serialize(const wxString& wxstr) { - return serialize(static_cast(wxstr.ToUTF8())); - } - If we allowed wxwidgets string to be serializable, all sorts of surprising things - would be serializable in surprising ways, because wxWidgets can convert all - sorts of things into strings that you were likely not expecting, in ways - unlikely to be machine independent, so you if you give an object to be - hashed that you have not provided some correct means for serializing, C++ is - apt to unhelpfully and unexpectedly turn it into a wxString, - - If you make wxStrings hashable, suprising things become hashable. - However, we do make the strange data structure provided by wxString.ToUTF8() hashable, - so that the wxString will not be implicitly hashable, but will be explicitly hashable. - */ - - // data structure containing a serialized signed integer. - template, int> = 0> - class userial : public std::span { - public: - std::array::digits + 6) / 7> bblob; - userial(T i) { - byte* p = &bblob[0] + sizeof(bblob); - *(--p) = i & 0x7f; - i >>= 7; - while (i != 0) { - *(--p) = (i & 0x7f) | 0x80; - i >>= 7; - } - assert(p >= &bblob[0]); - *static_cast*>(this) = std::span(p, &bblob[0] + sizeof(bblob));; - } - }; - - // data structure containing a serialized signed integer. - template, int> = 0> - class iserial : public std::span { - public: - std::array::digits + 7) / 7> bblob; - iserial(T i) { - // Throw away the repeated leading bits, and g - byte* p = &bblob[0] + sizeof(bblob); - unsigned count; - if (i < 0) { - size_t ui = i; - count = (std::numeric_limits::digits - std::countl_one(ui)) / 7; - } - else { - size_t ui = i; - count = (std::numeric_limits::digits - std::countl_zero(ui)) / 7; - } - *(--p) = i & 0x7f; - while (count-- != 0) { - i >>= 7; - *(--p) = (i & 0x7f) | 0x80; - } - assert(p >= &bblob[0]); - *static_cast*>(this) = std::span(p, &bblob[0] + sizeof(bblob));; - } - }; - - - // converts machine dependent representation of an integer - // into a span pointin at a compact machine independent representation of an integer - // The integer is split into seven bit nibbles in big endian order, with the high - // order bit of the byte indicating that more bytes are to come. - // for an unsigned integer, all high order bytes of the form 0x80 are left out. - // for a positive signed integer, the same, except that the first byte - // of what is left must have zero at bit 6 - // for a negative signed integer, all the 0xFF bytes are left out, except - // that the first byte of what is left must have a one bit at bit six. - // - // small numbers get compressed. - // primarily used by hash and hsh so that the same numbers on different - // machines will generate the same hash - template std::enable_if_t, ro::userial > - serialize(T i) { - return userial(i); - /* we don't need all deserialize functions to have the same name, - indeed they have to be distinct because serialized data contains - no type information, but for the sake of template code we need all - things that may be serialized to be serialized by the serialize - command, so that one template can deal with any - number of serializable types */ - } - template std::enable_if_t, ro::iserial >serialize(T i) { - return iserial(i); - /* we don't need all deserialize functions to have the same name, but for the sake of template code we need all - things that may be serialized to be serialized by the serialize command, so that one template can deal with any - number of serializable types */ - } - -// Turns a compact machine independent representation of an uninteger -// into a 64 bit signed integer - template std::enable_if_t, T > - deserialize(const byte* p) { - auto oldp = p; - T i; - if (*p & 0x40)i = -64; - else i = 0; - while (*p & 0x80) { - i = (i | (*p++ & 0x7F)) << 7; - } - if (p - oldp > (std::numeric_limits::digits + 6) / 7)throw BadDataException(); - return i | *p; - } - // Turns a compact machine independent representation of an integer - // into a 64 bit unsigned integer - template std::enable_if_t, T > - deserialize(const byte * p) { - auto oldp = p; - T i{ 0 }; - while (*p & 0x80) { - i = (i | (*p++ & 0x7F)) << 7; - } - if (p - oldp > 9)throw BadDataException(); - return i | *p; - } - - // Turns a compact machine independent representation of an integer - // into a 64 bit signed integer - template std::enable_if_t || is_standard_unsigned_integer, T > - deserialize(std::span g) { - byte* p = static_cast(&g[0]); - T i{ deserialize(p) }; - if (p > &g[0]+g.size())throw BadDataException(); - return i; - } - - /* - It will be about a thousand years before numbers larger than 64 bits - appear in valid well formed input, and bad data structures have to be - dealt with a much higher level that knows what the numbers mean, - and deals with them according to their meaning - - Until then the low level code will arbitrarily truncate numbers larger - than sixty four bits, but numbers larger than sixty four bits are - permissible in input, are valid at the lowest level. - - We return uint64_t, rather than uint_fast64_t to ensure that all - implementations misinterpret garbage and malicious input in the - same way. - We cannot protect against Machiavelli perverting the input, so we - don't try very hard to prevent Murphy perverting the input, - but we do try to prevent Machiavelli from perverting the input in - ways that will induce peers to disagree. - - We use an explicit narrow_cast, rather than simply declaring th - function to be uint64_t, in order to express the intent to uniformly - force possibly garbage data being deserialized to standardized - garbage. - - We protect against malicious and ill formed data would cause the - system to go off the rails at a point of the enemy's choosing, - and we protect against malicious and ill formed data that one party - might interpret in one way, and another party might interpret in a - different way. - - Ill formed data that just gets converted into well formed, but - nonsense data can cause no harm that well formed nonsense data - could not cause. - - It suffices, therefore, to ensure that all implementations misinterpret - input containing unreasonably large numbers as the same number. - - Very large numbers are valid in themselves, but not going to be valid - as part of valid data structures for a thousand years or so. - - The largest numbers occurring in well formed valid data will be - currency amounts, and the total number of the smallest unit of - currency is fixed at 2^64-1 which will suffice for a thousand years. - Or we might allow arbitrary precision floating point with powers of - a thousand, so that sensible numbers to a human are represented by - sensible numbers in the actual representation. - - secret keys, scalars are actually much larger numbers, modulo - 0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ecU - but they are represented in a different format, their binary format - being fixed size low endian format, as 256 bit numbers, though only - 253 bits are actually needed and used, and their human readable - format being 44 digits in a base 58 representation.*/ - - // a compile time test to check if an object class has a machine independent representation - template struct is_serializable{ - template - static constexpr decltype(ro::serialize(std::declval()), bool()) test() { - if constexpr (sizeof...(Args2) > 0) { - return is_serializable::template test(); - } - else { - return true; - } - } - template static constexpr bool test(int = 0) { - return false; - } - static constexpr bool value = is_serializable::template test(); - }; - - template - concept serializable = is_serializable::value; - - static_assert( !serializable - && serializable, char*, std::span>, - "concepts needed"); - - template ro::CompileSizedString< (2 * sizeof(T))>bin2hex(const T& pt) { - ro::CompileSizedString< (2 * sizeof(T))>sz; - sodium_bin2hex(&sz[0], sizeof(pt.blob) * 2 + 1, &pt.blob[0], pt.blob.size()); - return sz; - } - - template T hex2bin(const ro::CompileSizedString< (2 * sizeof(T))>& sz){ - T pt; - size_t bin_len{ sizeof(T) }; - sodium_hex2bin( - reinterpret_cast (&pt), - sizeof(T), - &sz[0], 2 * sizeof(T), - nullptr, &bin_len, nullptr - ); - return pt; - } - - template decltype(std::declval().blob, ro::CompileSizedString < (sizeof(T) * 8 + 5) / 6>()) to_base64_string(const T& p_blob) { - ro::CompileSizedString < (sizeof(T) * 8 + 5) / 6> sz; - bits2base64( - &(p_blob.blob[0]), 0, sizeof(p_blob.blob) * 8, - std::span(sz) - ); - return sz; - } - -} //End ro namespace namespace ristretto255 { using diff --git a/src/serialization.h b/src/serialization.h new file mode 100644 index 0000000..cae338e --- /dev/null +++ b/src/serialization.h @@ -0,0 +1,340 @@ +namespace ro { + + // Decay to pointer is dangerously convenient, + // but in some situations it is just convenient + // This class provides an std:array one larger + // than the compile time string size, which decays + // to char*, std::string, and wxString + // In some code, this is ambiguous, so casts + // must sometimes be explicitly invoked. + template + class CompileSizedString : public std::array{ + public: + static constexpr int length{ stringlen }; + CompileSizedString() { + *(this->rbegin()) = '0'; + } + CompileSizedString(char *psz) { + auto tsz{ this->rbegin() }; + *tsz = '0'; + if (psz != nullptr) { + auto usz = tsz + strlen; + while (tsz < usz && *psz != '\0') + *tsz++ = *psz++; + *tsz = '\0'; + } + } + operator char* () & { + char* pc = &(static_cast*>(this)->operator[](0)); + return pc; + } + + operator const char* () const& { + const char* pc = &(static_cast*>(this)->operator[](0)); + return pc; + } + operator const char* () const&& { + const char* pc = &(static_cast*>(this)->operator[](0)); + return pc; + } + operator std::string() const& { + return std::string((const char*)*this, this->length); + } + operator std::string() const&& { + return std::string((const char*)*this, this->length); + } + operator wxString() const& { + return wxString::FromUTF8Unchecked((const char*)(*this)); + } + operator std::span() const& { + return std::span(static_cast((char*)*this), stringlen + 1); + } + operator wxString() const&& { + return wxString::FromUTF8Unchecked((const char*)(*this)); + } + operator std::span() const&& { + return std::span(static_cast((char*)*this), stringlen + 1); + } + }; + + // This template generates a span over an indexable byte type, + // such as a C array or an std::array, but not pointers + template < typename T> + std::enable_if_t< + !std::is_pointer::value && + sizeof(std::declval()[0]) == 1, + std::span + > serialize(const T& a) { + return std::span(static_cast(static_cast(&a[0])), std::size(a)); + } + + // Compile time test to see if a type has a blob array member + // This can be used in if constexpr (is_blob_field_type::value) + // By convention, blob fields are an std::array of unsigned bytes + // therefore already serializable. + template struct is_blob_field_type{ + template static constexpr decltype(std::declval().blob.size(), bool()) test() { + return sizeof(std::declval().blob[0])==1; + } + template static constexpr bool test(int = 0) { + return false; + } + static constexpr bool value = is_blob_field_type::template test(); + }; + + template concept blob_type = ro::is_blob_field_type::value; + + + // At present our serial classes consist of std::span and custom classes that publicly inherit from std::span + // To handle compound objects, add custom classes inheriting from std::span[n] + + // template class that generates a std::span of bytes over the blob + // field of any object containing a blob record, which is normally sufficient + // for a machine independent representation of that object + template std::span serialize(const T& pt) { + return serialize(pt.blob); + } + + // method that assumes that any char * pointer points a null terminated string + // and generates a std::span of bytes over the string including the terminating + // null. + // we assume the string is already machine independent, which is to say, we assume + // it comes from a utf8 locale. + + inline auto serialize(const char* sp) { return std::span(static_cast(static_cast(sp)), strlen(sp) + 1); } + + inline auto serialize(const decltype(std::declval().ToUTF8()) sz){ + return serialize(static_cast(sz)); + } + /* + inline auto serialize(const wxString& wxstr) { + return serialize(static_cast(wxstr.ToUTF8())); + } + If we allowed wxwidgets string to be serializable, all sorts of surprising things + would be serializable in surprising ways, because wxWidgets can convert all + sorts of things into strings that you were likely not expecting, in ways + unlikely to be machine independent, so you if you give an object to be + hashed that you have not provided some correct means for serializing, C++ is + apt to unhelpfully and unexpectedly turn it into a wxString, + + If you make wxStrings hashable, suprising things become hashable. + However, we do make the strange data structure provided by wxString.ToUTF8() hashable, + so that the wxString will not be implicitly hashable, but will be explicitly hashable. + */ + + // data structure containing a serialized unsigned integer + // Converts an unsigned integer to VLQ format, and creates a bytespan pointing at it. + // VLQ format, Variable Length Quantity (It is a standard used by LLVM and others) + template class userial : public std::span { + public: + std::array::digits + 6) / 7> bblob; + userial(T i) { + byte* p = &bblob[0] + sizeof(bblob); + *(--p) = i & 0x7f; + i >>= 7; + while (i != 0) { + *(--p) = (i & 0x7f) | 0x80; + i >>= 7; + } + assert(p >= &bblob[0]); + *static_cast*>(this) = std::span(p, &bblob[0] + sizeof(bblob));; + } + }; + + // data structure containing a serialized signed integer, + // Converts an signed integer to VLQ format, and creates a bytespan pointing at it. + // VLQ format, Variable Length Quantity (It is a standard used by LLVM and others) + template class iserial : public std::span { + public: + std::array::digits + 7) / 7> bblob; + iserial(T i) { + // Throw away the repeated leading bits, and g + byte* p = &bblob[0] + sizeof(bblob); + unsigned count; + if (i < 0) { + size_t ui = i; + count = (std::numeric_limits::digits - std::countl_one(ui)) / 7; + } + else { + size_t ui = i; + count = (std::numeric_limits::digits - std::countl_zero(ui)) / 7; + } + *(--p) = i & 0x7f; + while (count-- != 0) { + i >>= 7; + *(--p) = (i & 0x7f) | 0x80; + } + assert(p >= &bblob[0]); + *static_cast*>(this) = std::span(p, &bblob[0] + sizeof(bblob));; + } + }; + + // converts machine dependent representation of an integer + // into a span pointin at a compact machine independent representation of an integer + // The integer is split into seven bit nibbles in big endian order + // (VLQ format), with the high + // order bit of the byte indicating that more bytes are to come. + // for an unsigned integer, all high order bytes of the form 0x80 are left out. + // for a positive signed integer, the same, except that the first byte + // of what is left must have zero at bit 6 + // for a negative signed integer, all the 0xFF bytes are left out, except + // that the first byte of what is left must have a one bit at bit six. + // + // small numbers get compressed. + // primarily used by hash and hsh so that the same numbers on different + // machines will generate the same hash + template userial serialize(T i) { + return userial(i); + /* we don't need all deserialize functions to have the same name, + indeed they have to be distinct because serialized data contains + no type information, but for the sake of template code we need all + things that may be serialized to be serialized by the serialize + command, so that one template can deal with any + number of serializable types */ + } + template iserial serialize(T i) { + return iserial(i); + /* we don't need all deserialize functions to have the same name, but for the sake of template code we need all + things that may be serialized to be serialized by the serialize command, so that one template can deal with any + number of serializable types */ + } + +// Turns a compact machine independent representation of an uninteger +// into a 64 bit signed integer + template T deserialize(const byte* p) { + auto oldp = p; + T i; + if (*p & 0x40)i = -64; + else i = 0; + while (*p & 0x80) { + i = (i | (*p++ & 0x7F)) << 7; + } + if (p - oldp > (std::numeric_limits::digits + 6) / 7)throw BadDataException(); + return i | *p; + } + // Turns a compact machine independent representation of an integer + // into a 64 bit unsigned integer + template T deserialize(const byte * p) { + auto oldp = p; + T i{ 0 }; + while (*p & 0x80) { + i = (i | (*p++ & 0x7F)) << 7; + } + if (p - oldp > 9)throw BadDataException(); + return i | *p; + } + + // Turns a compact machine independent representation of an integer + // into a 64 bit signed integer + template T deserialize(std::span g) { + byte* p = static_cast(&g[0]); + T i{ deserialize(p) }; + if (p > &g[0]+g.size())throw BadDataException(); + return i; + } + + /* + It will be about a thousand years before numbers larger than 64 bits + appear in valid well formed input, and bad data structures have to be + dealt with a much higher level that knows what the numbers mean, + and deals with them according to their meaning + + Until then the low level code will arbitrarily truncate numbers larger + than sixty four bits, but numbers larger than sixty four bits are + permissible in input, are valid at the lowest level. + + We return uint64_t, rather than uint_fast64_t to ensure that all + implementations misinterpret garbage and malicious input in the + same way. + We cannot protect against Machiavelli perverting the input, so we + don't try very hard to prevent Murphy perverting the input, + but we do try to prevent Machiavelli from perverting the input in + ways that will induce peers to disagree. + + We use an explicit narrow_cast, rather than simply declaring th + function to be uint64_t, in order to express the intent to uniformly + force possibly garbage data being deserialized to standardized + garbage. + + We protect against malicious and ill formed data would cause the + system to go off the rails at a point of the enemy's choosing, + and we protect against malicious and ill formed data that one party + might interpret in one way, and another party might interpret in a + different way. + + Ill formed data that just gets converted into well formed, but + nonsense data can cause no harm that well formed nonsense data + could not cause. + + It suffices, therefore, to ensure that all implementations misinterpret + input containing unreasonably large numbers as the same number. + + Very large numbers are valid in themselves, but not going to be valid + as part of valid data structures for a thousand years or so. + + The largest numbers occurring in well formed valid data will be + currency amounts, and the total number of the smallest unit of + currency is fixed at 2^64-1 which will suffice for a thousand years. + Or we might allow arbitrary precision floating point with powers of + a thousand, so that sensible numbers to a human are represented by + sensible numbers in the actual representation. + + secret keys, scalars are actually much larger numbers, modulo + 0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ecU + but they are represented in a different format, their binary format + being fixed size low endian format, as 256 bit numbers, though only + 253 bits are actually needed and used, and their human readable + format being 44 digits in a base 58 representation.*/ + + // a compile time test to check if an object class has a machine independent representation + template struct is_serializable{ + template + static constexpr decltype(ro::serialize(std::declval()), bool()) test() { + if constexpr (sizeof...(Args2) > 0) { + return is_serializable::template test(); + } + else { + return true; + } + } + template static constexpr bool test(int = 0) { + return false; + } + static constexpr bool value = is_serializable::template test(); + }; + + template + concept serializable = is_serializable::value; + + static_assert( !serializable + && serializable, char*, std::span>, + "concepts needed"); + + template ro::CompileSizedString< (2 * sizeof(T))>bin2hex(const T& pt) { + ro::CompileSizedString< (2 * sizeof(T))>sz; + sodium_bin2hex(&sz[0], sizeof(pt.blob) * 2 + 1, &pt.blob[0], pt.blob.size()); + return sz; + } + + template T hex2bin(const ro::CompileSizedString< (2 * sizeof(T))>& sz){ + T pt; + size_t bin_len{ sizeof(T) }; + sodium_hex2bin( + reinterpret_cast (&pt), + sizeof(T), + &sz[0], 2 * sizeof(T), + nullptr, &bin_len, nullptr + ); + return pt; + } + + template decltype(std::declval().blob, ro::CompileSizedString < (sizeof(T) * 8 + 5) / 6>()) to_base64_string(const T& p_blob) { + ro::CompileSizedString < (sizeof(T) * 8 + 5) / 6> sz; + bits2base64( + &(p_blob.blob[0]), 0, sizeof(p_blob.blob) * 8, + std::span(sz) + ); + return sz; + } + +} //End ro namespace \ No newline at end of file diff --git a/src/stdafx.h b/src/stdafx.h index a48208b..82ed699 100644 --- a/src/stdafx.h +++ b/src/stdafx.h @@ -92,6 +92,7 @@ static_assert(wxMAJOR_VERSION == 3 && wxMINOR_VERSION == 2 && wxRELEASE_NUMBER = #include "rotime.h" #include "slash6.h" #include "ISqlite3.h" +#include "serialization.h" #include "ristretto255.h" #include "secrets.h" #include "mpir_and_base58.h" From b16ddb2071b392d2e86613312fdc9c7cc63ffe42 Mon Sep 17 00:00:00 2001 From: Cheng Date: Wed, 27 Sep 2023 21:55:13 +1000 Subject: [PATCH 11/20] stashing changes here, and going back in time till file dialogs work --- .gitconfig | 3 ++- msvc/wallet.manifest | 6 ++++++ sqlite3/sqlite3.c | 3 +-- src/frame.cpp | 21 ++++++++++++--------- src/serialization.h | 6 +++++- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.gitconfig b/.gitconfig index 0205c94..53906c7 100644 --- a/.gitconfig +++ b/.gitconfig @@ -8,7 +8,8 @@ [alias] lg = log --max-count=6 --pretty=format:'%C(auto)%h %d %Creset %p %C("#60A0FF")%cr %Cgreen %cn %G? %GT trust%Creset%n %<(78,trunc)%s%n' graph = log --max-count=38 --graph --pretty=format:'%C(auto)%h %<(78,trunc)%s %Cgreen(%cr) %C(bold blue)%cn %G?%Creset' --abbrev-commit - alias = ! git config --get-regexp ^alias\\. | sed -e s/^alias\\.// -e s/\\ /\\ =\\ / | grep -v ^'alias ' | sort + alias = !git config --get-regexp ^alias\\. | sed -e s/^alias\\.// -e s/\\ /\\ =\\ / | grep -v ^'alias ' | sort + utcmt = commit --date=\"$(date --utc +%Y-%m-%dT%H:%M:%Sz)\" [commit] gpgSign = true [push] diff --git a/msvc/wallet.manifest b/msvc/wallet.manifest index b9de604..984fdd6 100644 --- a/msvc/wallet.manifest +++ b/msvc/wallet.manifest @@ -1,5 +1,11 @@ + + + + + + UTF-8 diff --git a/sqlite3/sqlite3.c b/sqlite3/sqlite3.c index b90ba45..1f2cb24 100644 --- a/sqlite3/sqlite3.c +++ b/sqlite3/sqlite3.c @@ -28,12 +28,11 @@ #define SQLITE_THREADSAFE 2 //Sets the default mode to SQLITE_CONFIG_MULTITHREAD. One thread, one database connection. Data structures such as compiled SQL are threadlocal. But sqlite3 is empowered to do its own multithreading. Many databases per database connection. Database connection and compiled sql statements are threadlocal. last_insert_rowid() is not subject to race conditions in this mode. #define SQLITE_DEFAULT_MEMSTATUS 0 //Don't track memory usage. Disables the ability of the program using sqlite3 to monitor its memory usage. This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster. #define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1 // in WAL mode, recent changes to the database might be rolled back by a power loss, but the database will not be corrupted. Furthermore, transaction commit is much faster in WAL mode using synchronous=NORMAL than with the default synchronous=FULL. For these reasons, it is recommended that the synchronous setting be changed from FULL to NORMAL when switching to WAL mode. This compile-time option will accomplish that. -#define SQLITE_DEFAULT_FOREIGN_KEYS 0 //Dont handle foreign key constraints. Programmer has to do it himself. +#define SQLITE_DEFAULT_FOREIGN_KEYS 1 // Enforce foreign key constraints. #define SQLITE_LIKE_DOESNT_MATCH_BLOBS 1 //Blobs are not strings. Historically, SQLite has allowed BLOB operands to the LIKE and GLOB operators. But having a BLOB as an operand of LIKE or GLOB complicates and slows the LIKE optimization. When this option is set, it means that the LIKE and GLOB operators always return FALSE if either operand is a BLOB. That simplifies the implementation of the LIKE optimization and allows queries that use the LIKE optimization to run faster. #define SQLITE_MAX_EXPR_DEPTH 0 //Setting the maximum expression parse-tree depth to zero disables all checking of the expression parse-tree depth, which simplifies the code resulting in faster execution, and helps the parse tree to use less memory. #define SQLITE_OMIT_DECLTYPE 1 // By omitting the (seldom-needed) ability to return the declared type of columns from the result set of query, prepared statements can be made to consume less memory. #define SQLITE_OMIT_DEPRECATED 1 -#define SQLITE_DQS 0 //Don't accept double quoted string literals. #define SQLITE_OMIT_PROGRESS_CALLBACK 1 #define SQLITE_OMIT_SHARED_CACHE 1 #define SQLITE_OMIT_UTF16 1 diff --git a/src/frame.cpp b/src/frame.cpp index 6708f5c..0847167 100644 --- a/src/frame.cpp +++ b/src/frame.cpp @@ -249,11 +249,6 @@ void Frame::NewWallet(wxFileName& filename, ristretto255::hash<256>& secret) { PRAGMA journal_mode = WAL; PRAGMA synchronous = 1; BEGIN IMMEDIATE TRANSACTION; -CREATE TABLE "Keys"( - "ROWID" INTEGER PRIMARY KEY, - "pubkey" BLOB NOT NULL UNIQUE, - "id" integer NOT NULL, - "use" INTEGER NOT NULL); CREATE UNIQUE INDEX i_pubkey ON Keys (pubkey); CREATE UNIQUE INDEX i_id ON Keys (use, id); @@ -261,16 +256,24 @@ CREATE UNIQUE INDEX i_id ON Keys (use, id); CREATE TABLE "Names"( "ROWID" INTEGER PRIMARY KEY, "name" TEXT NOT NULL UNIQUE -); +) STRICT; CREATE UNIQUE INDEX i_names ON Names (name); CREATE TABLE "Misc"( "ROWID" INTEGER PRIMARY KEY, - "m" BLOB -); + "m" ANY +) STRICT; + +CREATE TABLE "Keys"( + "ROWID" INTEGER PRIMARY KEY, + "pubkey" BLOB NOT NULL UNIQUE, + "id" integer NOT NULL, + "use" INTEGER NOT NULL, + FOREIGN KEY(id) REFERENCES Names(ROWID) +) STRICT; + COMMIT;)|"); - wxLogMessage("\t\tConstructing default wallet %s", filename.GetFullPath()); // We now have a working wallet file with no valid data. Attempting to create a strong random secret, a name, and public and private keys for that name. wxLogMessage("\t\tGenerating random 128 bit wallet secret"); auto text_secret{ DeriveTextSecret(ristretto255::scalar::random(), 1) }; diff --git a/src/serialization.h b/src/serialization.h index cae338e..a06cb6e 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -1,4 +1,4 @@ -namespace ro { +namespace ro { // Decay to pointer is dangerously convenient, // but in some situations it is just convenient @@ -125,6 +125,10 @@ namespace ro { // data structure containing a serialized unsigned integer // Converts an unsigned integer to VLQ format, and creates a bytespan pointing at it. // VLQ format, Variable Length Quantity (It is a standard used by LLVM and others) + // On reflection, VLQ format is not convenient for the intended usage (merkle patricia trees + // representing SQL indexes, and a better format is to compress leading zero or leading 0xFF bytes + // with the length of the run being implied by a count of the bytes following the run) + template class userial : public std::span { public: std::array::digits + 6) / 7> bblob; From b60c57c2d26e80d0ef56f319c29df4a9730be763 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 28 Sep 2023 02:13:08 +0000 Subject: [PATCH 12/20] rolling back wx widgets to v3.2.0 file dialog now works, but bugs in my sql code --- src/stdafx.h | 9 +++++++-- wxWidgets | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/stdafx.h b/src/stdafx.h index 82ed699..c8feed3 100644 --- a/src/stdafx.h +++ b/src/stdafx.h @@ -71,7 +71,7 @@ static_assert(wxUSE_STD_CONTAINERS == 1, "wxWidgets api out of date"); static_assert(wxUSE_STD_STRING_CONV_IN_WXSTRING == 1, "want wxString to conform to std::string"); static_assert(wxUSE_SECRETSTORE >0, "need wxSecretStore to actually work"); // The api may be there, but will always return false if wxUSE_SECRETSTORE is zero -static_assert(wxUSE_STD_DEFAULT == 1 && wxUSE_STL==1); +//static_assert(wxUSE_STD_DEFAULT == 1 && wxUSE_STL==1); std::span& operator^=(std::span&, byte *); #ifndef wxHAS_IMAGES_IN_RESOURCES #include "rho.xpm" //Defines the icon AAArho on linux @@ -88,7 +88,12 @@ inline wxString _wx(const char* sz) { return wxString::FromUTF8Unchecked(sz); } // it provides an array spaceship operator that solves the inconsistency // problem in a better way. Spaceship compares of arrays in my code are // no longer ill formed. -static_assert(wxMAJOR_VERSION == 3 && wxMINOR_VERSION == 2 && wxRELEASE_NUMBER == 2 && wxSUBRELEASE_NUMBER == 1 && wxVERSION_STRING == wxT("wxWidgets 3.2.2.1"), "expecting wxWidgets 3.2.2.1"); +static_assert(wxMAJOR_VERSION == 3, "expecting wxWidgets wxWidgets 3.2.0"); +static_assert(wxMINOR_VERSION == 2, "expecting wxWidgets wxWidgets 3.2.0"); +static_assert(wxRELEASE_NUMBER == 0, "expecting wxWidgets wxWidgets 3.2.0"); +static_assert(wxSUBRELEASE_NUMBER == 1, "expecting wxWidgets wxWidgets 3.2.0"); +static_assert(wxVERSION_STRING == wxT("wxWidgets 3.2.0"), "expecting wxWidgets wxWidgets 3.2.0"); +// static_assert(wxMAJOR_VERSION == 3 && wxMINOR_VERSION == 2 && wxRELEASE_NUMBER == 0 && wxSUBRELEASE_NUMBER == 1 && wxVERSION_STRING == wxT("wxWidgets 3.2.0"), "expecting wxWidgets wxWidgets 3.2.0"); #include "rotime.h" #include "slash6.h" #include "ISqlite3.h" diff --git a/wxWidgets b/wxWidgets index 8ef067c..78cd5a0 160000 --- a/wxWidgets +++ b/wxWidgets @@ -1 +1 @@ -Subproject commit 8ef067c7317b302068f1b774c92c5250052c2a53 +Subproject commit 78cd5a04e6b87a0d3643ba03c8f28ccc1ce2f051 From a68282e390802572acfef25b7ccdb9192282a125 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 28 Sep 2023 04:34:51 +0000 Subject: [PATCH 13/20] modified: .gitconfig --- .gitconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitconfig b/.gitconfig index 53906c7..b520074 100644 --- a/.gitconfig +++ b/.gitconfig @@ -9,7 +9,7 @@ lg = log --max-count=6 --pretty=format:'%C(auto)%h %d %Creset %p %C("#60A0FF")%cr %Cgreen %cn %G? %GT trust%Creset%n %<(78,trunc)%s%n' graph = log --max-count=38 --graph --pretty=format:'%C(auto)%h %<(78,trunc)%s %Cgreen(%cr) %C(bold blue)%cn %G?%Creset' --abbrev-commit alias = !git config --get-regexp ^alias\\. | sed -e s/^alias\\.// -e s/\\ /\\ =\\ / | grep -v ^'alias ' | sort - utcmt = commit --date=\"$(date --utc +%Y-%m-%dT%H:%M:%Sz)\" + utcmt = !git commit --date=\"$(date --utc +%Y-%m-%dT%H:%M:%Sz)\" [commit] gpgSign = true [push] From d82d5218bc0f1f623b5346a8691247622209e5b2 Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 28 Sep 2023 08:14:32 +0000 Subject: [PATCH 14/20] Got it working, the mystery crash was my update to wxWidgets 3.2.2.1 --- .gitconfig | 2 +- msvc/wallet.vcxproj | 4 ++-- sqlite3/sqlite3.c | 2 +- src/frame.cpp | 15 ++++++--------- wxWidgets | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.gitconfig b/.gitconfig index b520074..07f00e5 100644 --- a/.gitconfig +++ b/.gitconfig @@ -7,7 +7,7 @@ ignoreWhitespace = no [alias] lg = log --max-count=6 --pretty=format:'%C(auto)%h %d %Creset %p %C("#60A0FF")%cr %Cgreen %cn %G? %GT trust%Creset%n %<(78,trunc)%s%n' - graph = log --max-count=38 --graph --pretty=format:'%C(auto)%h %<(78,trunc)%s %Cgreen(%cr) %C(bold blue)%cn %G?%Creset' --abbrev-commit + graph = log --graph --pretty=format:'%C(auto)%h %<(78,trunc)%s %Cgreen(%cr) %C(bold blue)%cn %G?%Creset' --abbrev-commit alias = !git config --get-regexp ^alias\\. | sed -e s/^alias\\.// -e s/\\ /\\ =\\ / | grep -v ^'alias ' | sort utcmt = !git commit --date=\"$(date --utc +%Y-%m-%dT%H:%M:%Sz)\" [commit] diff --git a/msvc/wallet.vcxproj b/msvc/wallet.vcxproj index 99865da..41f374e 100644 --- a/msvc/wallet.vcxproj +++ b/msvc/wallet.vcxproj @@ -144,8 +144,8 @@ - - + + diff --git a/sqlite3/sqlite3.c b/sqlite3/sqlite3.c index 1f2cb24..5154108 100644 --- a/sqlite3/sqlite3.c +++ b/sqlite3/sqlite3.c @@ -28,7 +28,7 @@ #define SQLITE_THREADSAFE 2 //Sets the default mode to SQLITE_CONFIG_MULTITHREAD. One thread, one database connection. Data structures such as compiled SQL are threadlocal. But sqlite3 is empowered to do its own multithreading. Many databases per database connection. Database connection and compiled sql statements are threadlocal. last_insert_rowid() is not subject to race conditions in this mode. #define SQLITE_DEFAULT_MEMSTATUS 0 //Don't track memory usage. Disables the ability of the program using sqlite3 to monitor its memory usage. This setting causes the sqlite3_status() interfaces that track memory usage to be disabled. This helps the sqlite3_malloc() routines run much faster, and since SQLite uses sqlite3_malloc() internally, this helps to make the entire library faster. #define SQLITE_DEFAULT_WAL_SYNCHRONOUS 1 // in WAL mode, recent changes to the database might be rolled back by a power loss, but the database will not be corrupted. Furthermore, transaction commit is much faster in WAL mode using synchronous=NORMAL than with the default synchronous=FULL. For these reasons, it is recommended that the synchronous setting be changed from FULL to NORMAL when switching to WAL mode. This compile-time option will accomplish that. -#define SQLITE_DEFAULT_FOREIGN_KEYS 1 // Enforce foreign key constraints. +#define SQLITE_DEFAULT_FOREIGN_KEYS 0 //Dont handle foreign key constraints. Programmer has to do it himself. #define SQLITE_LIKE_DOESNT_MATCH_BLOBS 1 //Blobs are not strings. Historically, SQLite has allowed BLOB operands to the LIKE and GLOB operators. But having a BLOB as an operand of LIKE or GLOB complicates and slows the LIKE optimization. When this option is set, it means that the LIKE and GLOB operators always return FALSE if either operand is a BLOB. That simplifies the implementation of the LIKE optimization and allows queries that use the LIKE optimization to run faster. #define SQLITE_MAX_EXPR_DEPTH 0 //Setting the maximum expression parse-tree depth to zero disables all checking of the expression parse-tree depth, which simplifies the code resulting in faster execution, and helps the parse tree to use less memory. #define SQLITE_OMIT_DECLTYPE 1 // By omitting the (seldom-needed) ability to return the declared type of columns from the result set of query, prepared statements can be made to consume less memory. diff --git a/src/frame.cpp b/src/frame.cpp index 0847167..1696e64 100644 --- a/src/frame.cpp +++ b/src/frame.cpp @@ -249,6 +249,11 @@ void Frame::NewWallet(wxFileName& filename, ristretto255::hash<256>& secret) { PRAGMA journal_mode = WAL; PRAGMA synchronous = 1; BEGIN IMMEDIATE TRANSACTION; +CREATE TABLE "Keys"( + "ROWID" INTEGER PRIMARY KEY, + "pubkey" BLOB NOT NULL UNIQUE, + "id" integer NOT NULL, + "use" INTEGER NOT NULL) STRICT; CREATE UNIQUE INDEX i_pubkey ON Keys (pubkey); CREATE UNIQUE INDEX i_id ON Keys (use, id); @@ -264,16 +269,8 @@ CREATE TABLE "Misc"( "ROWID" INTEGER PRIMARY KEY, "m" ANY ) STRICT; - -CREATE TABLE "Keys"( - "ROWID" INTEGER PRIMARY KEY, - "pubkey" BLOB NOT NULL UNIQUE, - "id" integer NOT NULL, - "use" INTEGER NOT NULL, - FOREIGN KEY(id) REFERENCES Names(ROWID) -) STRICT; - COMMIT;)|"); + wxLogMessage("\t\tConstructing default wallet %s", filename.GetFullPath()); // We now have a working wallet file with no valid data. Attempting to create a strong random secret, a name, and public and private keys for that name. wxLogMessage("\t\tGenerating random 128 bit wallet secret"); auto text_secret{ DeriveTextSecret(ristretto255::scalar::random(), 1) }; diff --git a/wxWidgets b/wxWidgets index 78cd5a0..c2980fa 160000 --- a/wxWidgets +++ b/wxWidgets @@ -1 +1 @@ -Subproject commit 78cd5a04e6b87a0d3643ba03c8f28ccc1ce2f051 +Subproject commit c2980fa75c41eac5152898fc41709832ccb24759 From bba593180e36f7356fa52cd64366eabf3363165a Mon Sep 17 00:00:00 2001 From: Cheng Date: Thu, 28 Sep 2023 22:14:29 +0000 Subject: [PATCH 15/20] displays names in alphabetic order sanity test of pubkey mysteriously fails in display_wallet, yet identical test with same values succeeds in unit_test need to create a view once sanity test passes. then need to refresh display on edit/add name need to make a second try at integrating release v3.2.2.1 --- src/ILog.cpp | 7 +++ src/ILog.h | 3 + src/display_wallet.cpp | 140 +++++++++++++++++++++++------------------ src/unit_test.cpp | 12 ++-- 4 files changed, 96 insertions(+), 66 deletions(-) diff --git a/src/ILog.cpp b/src/ILog.cpp index 6ef3132..c7ddca7 100644 --- a/src/ILog.cpp +++ b/src/ILog.cpp @@ -49,3 +49,10 @@ MyException::MyException(const char* sz, int i, const char* func__, const char* err = std::format(R"|({} line {}, function {}, file {})|", sz, i, func__, FILE__); } + +MyException::MyException(MyException e, int i, const char* func, const char* file) noexcept : + err(std::format(R"|({} +line {}, function {}, file {})|", e.what(), i, func, file)), err_number(e.what_num()) {} +MyException::MyException(std::exception e, int i, const char* func, const char* file) noexcept: + err(std::format(R"|({} +line {}, function {}, file {})|", e.what(), i, func, file)), err_number(i) {} \ No newline at end of file diff --git a/src/ILog.h b/src/ILog.h index 01bdc7d..7aea7cc 100644 --- a/src/ILog.h +++ b/src/ILog.h @@ -32,6 +32,9 @@ public: virtual const int what_num() const { return err_number; } + explicit MyException(MyException e, std::string str)noexcept : err(e.what() + str), err_number(e.what_num()) {} + explicit MyException(MyException e, int i, const char* func, const char* file) noexcept; + explicit MyException(std::exception e, int i, const char* func, const char* file) noexcept; }; class FatalException : public MyException { diff --git a/src/display_wallet.cpp b/src/display_wallet.cpp index 11a6efa..b10a09b 100644 --- a/src/display_wallet.cpp +++ b/src/display_wallet.cpp @@ -1,72 +1,92 @@ #include "stdafx.h" using ro::base58; +static constexpr char SrcFilename[]{ "src/display_wallet.cpp" }; display_wallet::display_wallet(wxWindow* parent, wxFileName& walletfile) : wxPanel(parent, myID_WALLET_UI, wxDefaultPosition, wxDefaultSize, wxTAB_TRAVERSAL, wxT("Wallet")), m_db(nullptr), m_menuitem_add_name(this, &display_wallet::add_name_event_handler) { wxLogMessage(wxT("Loading %s"), walletfile.GetFullPath()); - if (!walletfile.IsOk() || !walletfile.HasName() || !walletfile.HasExt()) throw MyException("unexpected file name"); - if (!walletfile.FileExists())throw MyException( - walletfile.GetFullPath().append(" does not exist.").ToUTF8() - ); - m_db.reset(Sqlite3_open(walletfile.GetFullPath().ToUTF8())); - sql_read_from_misc read_from_misc(m_db); - if (!read_from_misc(1) || read_from_misc.value() != WALLET_FILE_IDENTIFIER)throw MyException(sz_unrecognizable_wallet_file_format); - if (!read_from_misc(2) || read_from_misc.value() != WALLET_FILE_SCHEMA_VERSION_0_0 || !read_from_misc(4))throw MyException(sz_unrecognized_wallet_schema); - read_from_misc.read(m_MasterSecret); - if (!m_MasterSecret.valid()) throw MyException(sz_cold_wallets_not_yet_implemented); - auto sizer = new wxBoxSizer(wxHORIZONTAL); - m_lSizer = new wxBoxSizer(wxVERTICAL); - m_rSizer = new wxBoxSizer(wxVERTICAL); - sizer->Add(m_lSizer,0, wxGROW, 4); - sizer->Add(m_rSizer, 50, wxGROW, 4); - SetSizer(sizer); - ro::sql read_keys(m_db, R"|(SELECT * FROM "Keys";)|"); - sql_read_name read_name(m_db); //*It would be better to have a select statement goes through the name table, in name order. This is unit test code wrongly repurposed. - /* ro::sql sql_read_names( - m_db, - R"|(SELECT Names.name, Keys.pubkey FROM Names INNER JOIN Keys ON Names.ROWID=Keys.id AND Keys.use=1 ORDER BY Names.name;)|"){} */ - // m_db.reset(nullptr);// Force error of premature destruction of Isqlite3 - while (read_keys.step() == Icompiled_sql::ROW) { - auto pubkey = read_keys.column(1); - auto id = read_keys.column(2); - auto use = read_keys.column(3); - if (use != 1)throw MyException(sz_unknown_secret_key_algorithm); - if (!read_name(id)) throw MyException(sz_no_corresponding_entry); - const char* name = read_name.name(); - if (m_MasterSecret(name).timesBase() != *pubkey)throw MyException(std::string(sz_public_key_of) + name + sz_fails_to_correspond); - m_lSizer->Add( - new wxStaticText( - this, - wxID_ANY, - name, - wxDefaultPosition, wxDefaultSize, wxALIGN_RIGHT|wxST_ELLIPSIZE_END - ), - 10, - wxEXPAND | // make horizontally stretchable - wxALL, // and make border all around - 2); - m_rSizer->Add( - new wxStaticText( - this, - wxID_ANY, - "#" + base58(*pubkey).operator std::string(), - wxDefaultPosition, wxDefaultSize, wxALIGN_LEFT | wxST_ELLIPSIZE_END - ), - 10, - wxEXPAND | // make horizontally stretchable - wxALL, // and make border all around - 2); - } - Bind(wxEVT_CLOSE_WINDOW, &display_wallet::OnClose, this); - this->SetSize(this->GetParent()->GetClientSize()); - singletonFrame->m_LastUsedSqlite.Assign(walletfile); + try { + if (!walletfile.IsOk() || !walletfile.HasName() || !walletfile.HasExt()) throw MyException("unexpected file name", __LINE__, __func__, SrcFilename); + if (!walletfile.FileExists())throw MyException( + walletfile.GetFullPath().append(" does not exist.").ToUTF8(), + __LINE__, __func__, SrcFilename); + m_db.reset(Sqlite3_open(walletfile.GetFullPath().ToUTF8())); + sql_read_from_misc read_from_misc(m_db); + if (!read_from_misc(1) || read_from_misc.value() != WALLET_FILE_IDENTIFIER)throw MyException(sz_unrecognizable_wallet_file_format); + if (!read_from_misc(2) || read_from_misc.value() != WALLET_FILE_SCHEMA_VERSION_0_0 || !read_from_misc(4))throw MyException(sz_unrecognized_wallet_schema); + m_MasterSecret= *read_from_misc.value(); + ILogMessage(std::format("\t\tmaster secret: #{}", ro::base58(m_MasterSecret).operator const char* ()).c_str()); + if(!m_MasterSecret.valid()) throw MyException(sz_cold_wallets_not_yet_implemented, __LINE__, __func__, SrcFilename); + auto sizer = new wxBoxSizer(wxHORIZONTAL); + m_lSizer = new wxBoxSizer(wxVERTICAL); + m_rSizer = new wxBoxSizer(wxVERTICAL); + sizer->Add(m_lSizer, 0, wxGROW, 4); + sizer->Add(m_rSizer, 50, wxGROW, 4); + SetSizer(sizer); + try { + ro::sql read_names_and_keys(m_db, + R"|(SELECT Names.name AS name, Keys.pubkey AS pubkey FROM Names INNER JOIN Keys ON Names.ROWID=Keys.id AND Keys.use=1 ORDER BY LOWER(name), name COLLATE NOCASE;)|"); + while (read_names_and_keys.step() == Icompiled_sql::ROW) { + std::string name = read_names_and_keys.column(0); + auto pubkey = *read_names_and_keys.column(1); +// if (m_MasterSecret(name).timesBase() != pubkey)throw MyException(std::string(sz_public_key_of) + name + sz_fails_to_correspond); + m_lSizer->Add( + new wxStaticText( + this, + wxID_ANY, + name, + wxDefaultPosition, wxDefaultSize, wxALIGN_RIGHT | wxST_ELLIPSIZE_END + ), + 10, + wxEXPAND | // make horizontally stretchable + wxALL, // and make border all around + 2); + m_rSizer->Add( + new wxStaticText( + this, + wxID_ANY, + "#" + base58(pubkey).operator std::string(), + wxDefaultPosition, wxDefaultSize, wxALIGN_LEFT | wxST_ELLIPSIZE_END + ), + 10, + wxEXPAND | // make horizontally stretchable + wxALL, // and make border all around + 2); + } + } + catch (const MyException& e) { + throw MyException(e, __LINE__, __func__, SrcFilename); + // sql exceptions tend to be unintelligible and excessively generic + // because unclear what statement provoked them. + } + catch (const std::exception& e) { + throw MyException(e, __LINE__, __func__, SrcFilename); + } + catch (...) { + szError = sz_unknown_error; + throw MyException(sz_unknown_error, __LINE__, __func__, SrcFilename); + } + Bind(wxEVT_CLOSE_WINDOW, &display_wallet::OnClose, this); + this->SetSize(this->GetParent()->GetClientSize()); + singletonFrame->m_LastUsedSqlite.Assign(walletfile); - wxMenu* menuFile{ singletonFrame->GetMenuBar()->GetMenu(0) }; - singletonFrame->GetMenuBar()->EnableTop(1, true); //enable edit menu. - wxMenu* menuEdit{ singletonFrame->GetMenuBar()->GetMenu(1) }; - m_menuitem_add_name.Insert(menuEdit, 0, "add name", "create new Zooko identity"); + wxMenu* menuFile{ singletonFrame->GetMenuBar()->GetMenu(0) }; + singletonFrame->GetMenuBar()->EnableTop(1, true); //enable edit menu. + wxMenu* menuEdit{ singletonFrame->GetMenuBar()->GetMenu(1) }; + m_menuitem_add_name.Insert(menuEdit, 0, "add name", "create new Zooko identity"); + } + catch (const MyException&) { + throw; + } + catch (const std::exception& e) { + throw MyException(e, __LINE__, __func__, SrcFilename); + } + catch (...) { + szError = sz_unknown_error; + throw MyException(sz_unknown_error, __LINE__, __func__, SrcFilename); + } } diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 989106c..8387083 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -241,9 +241,8 @@ static bool checkDataConversionsProduceExpected(void){ hash<256> hash_b{ hash_a,str_hash_a,scl_a,str_sclr_a,pt_a,str_pt_a,33, 66ull }; if (base58(hash_b).operator std::string() != "i22EVNPsKRjdxYTZrPPu9mx6vnrBjosFix5F4gn2mb2kF" - ){ - throw MyException("unexpected hash of transformations", __LINE__, __func__, SrcFilename); - } + ){ throw MyException("unexpected hash of transformations", __LINE__, __func__, SrcFilename); + } } catch (const MyException& e) { errorCode = e.what_num(); @@ -408,18 +407,19 @@ static bool OpenWallet(void) { if(!read_from_misc(2) || read_from_misc.value() != WALLET_FILE_SCHEMA_VERSION_0_0)throw MyException("Unrecognized wallet schema", __LINE__, __func__, SrcFilename); if (!read_from_misc(4)) throw MyException("Mastersecret missing", __LINE__, __func__, SrcFilename); ristretto255::CMasterSecret MasterSecret(*read_from_misc.value()); + ILogMessage(std::format("\t\tmaster secret: #{}", ro::base58(MasterSecret).operator const char *()).c_str()); ro::sql read_keys(db.get(), R"|(SELECT * FROM "Keys" LIMIT 5;)|"); sql_read_name read_name(db.get()); // db.reset(nullptr);// Force error of premature destruction of Isqlite3 while (read_keys.step() == Icompiled_sql::ROW) { - auto pubkey = read_keys.column(1); + auto pubkey = *read_keys.column(1); auto id = read_keys.column(2); auto use = read_keys.column(3); if (use != 1)throw MyException(sz_unknown_secret_key_algorithm, __LINE__, __func__, SrcFilename); if (!read_name(id)) throw MyException(sz_no_corresponding_entry, __LINE__, __func__, SrcFilename); const char* name = read_name.name(); - if(MasterSecret(name).timesBase()!=*pubkey)throw MyException(R"|(Public key of name fails to correspond)|", __LINE__, __func__, SrcFilename); - wxLogMessage(wxT("\t\t\"%s\" has expected public key 0x%s"), name, (wxString)(bin2hex(*pubkey))); + if(MasterSecret(name).timesBase()!=pubkey)throw MyException(R"|(Public key of name fails to correspond)|", __LINE__, __func__, SrcFilename); + wxLogMessage(wxT("\t\t\"%s\" has expected public key #%s"), name, (wxString)(ro::base58(pubkey).operator const char* ())); } } else { From 883d8c5b51a948487a7d62be33ee2aa45c9e4899 Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 29 Sep 2023 08:44:49 +0000 Subject: [PATCH 16/20] Found the sanity test bug. My hashing code incorrectly ignored the trailing null in std::strings. (but not in c strings) --- src/display_wallet.cpp | 4 ++-- src/serialization.h | 12 +++++++++++- src/unit_test.cpp | 9 ++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/display_wallet.cpp b/src/display_wallet.cpp index b10a09b..9c52a99 100644 --- a/src/display_wallet.cpp +++ b/src/display_wallet.cpp @@ -27,11 +27,11 @@ display_wallet::display_wallet(wxWindow* parent, wxFileName& walletfile) : SetSizer(sizer); try { ro::sql read_names_and_keys(m_db, - R"|(SELECT Names.name AS name, Keys.pubkey AS pubkey FROM Names INNER JOIN Keys ON Names.ROWID=Keys.id AND Keys.use=1 ORDER BY LOWER(name), name COLLATE NOCASE;)|"); + R"|(SELECT Names.name AS name, Keys.pubkey AS pubkey FROM Names INNER JOIN Keys ON Names.ROWID=Keys.id AND Keys.use=1 ORDER BY LOWER(name), name COLLATE BINARY;)|"); while (read_names_and_keys.step() == Icompiled_sql::ROW) { std::string name = read_names_and_keys.column(0); auto pubkey = *read_names_and_keys.column(1); -// if (m_MasterSecret(name).timesBase() != pubkey)throw MyException(std::string(sz_public_key_of) + name + sz_fails_to_correspond); + if (m_MasterSecret(name).timesBase() != pubkey)throw MyException(std::string(sz_public_key_of) + name + sz_fails_to_correspond); m_lSizer->Add( new wxStaticText( this, diff --git a/src/serialization.h b/src/serialization.h index a06cb6e..c35a143 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -65,7 +65,17 @@ sizeof(std::declval()[0]) == 1, std::span > serialize(const T& a) { - return std::span(static_cast(static_cast(&a[0])), std::size(a)); + int l; + const void* pt; + if constexpr (std::is_same_v, std::string>) { + l = a.length() + 1; + pt = a.c_str(); + } + else { + l = std::size(a); + pt = &a[0]; + } + return std::span(static_cast(pt), l); } // Compile time test to see if a type has a blob array member diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 8387083..47403d5 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -239,9 +239,12 @@ static bool checkDataConversionsProduceExpected(void){ std::string str_pt_a = &(base58(pt_a))[0]; assert(base58::bin(str_pt_a.c_str()) == pt_a); hash<256> hash_b{ hash_a,str_hash_a,scl_a,str_sclr_a,pt_a,str_pt_a,33, 66ull }; - if (base58(hash_b).operator std::string() != - "i22EVNPsKRjdxYTZrPPu9mx6vnrBjosFix5F4gn2mb2kF" - ){ throw MyException("unexpected hash of transformations", __LINE__, __func__, SrcFilename); + auto str_b = base58(hash_b).operator std::string(); + if (str_b != "7cTScjKyUtmbvc28BV3ok51szgrQmaBa2YE5HPBcukC9e" + ) { + throw MyException( + std::format("unexpected hash of transformations: hash256#{}", str_b), + __LINE__, __func__, SrcFilename); } } catch (const MyException& e) { From 5f4fe3104b062a687f337d5e6e2ed9d6d5429dde Mon Sep 17 00:00:00 2001 From: Cheng Date: Fri, 29 Sep 2023 11:16:13 +0000 Subject: [PATCH 17/20] Moved the compiled sql statments from the stack to unique pointers that are members of the window --- src/display_wallet.cpp | 19 ++++++++++--------- src/display_wallet.h | 2 ++ src/serialization.h | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/display_wallet.cpp b/src/display_wallet.cpp index 9c52a99..74502fc 100644 --- a/src/display_wallet.cpp +++ b/src/display_wallet.cpp @@ -13,10 +13,13 @@ display_wallet::display_wallet(wxWindow* parent, wxFileName& walletfile) : walletfile.GetFullPath().append(" does not exist.").ToUTF8(), __LINE__, __func__, SrcFilename); m_db.reset(Sqlite3_open(walletfile.GetFullPath().ToUTF8())); - sql_read_from_misc read_from_misc(m_db); - if (!read_from_misc(1) || read_from_misc.value() != WALLET_FILE_IDENTIFIER)throw MyException(sz_unrecognizable_wallet_file_format); - if (!read_from_misc(2) || read_from_misc.value() != WALLET_FILE_SCHEMA_VERSION_0_0 || !read_from_misc(4))throw MyException(sz_unrecognized_wallet_schema); - m_MasterSecret= *read_from_misc.value(); + m_read_from_misc.reset(new sql_read_from_misc(m_db)); + m_read_names_and_keys.reset(new ro::sql(m_db, + R"|(SELECT "Names".name AS name, "Keys".pubkey AS pubkey FROM "Names" INNER JOIN "Keys" )|" + R"|(ON "Names"."ROWID"="Keys".id AND "Keys".use=1 ORDER BY LOWER(name), name COLLATE BINARY;)|")); + if (!(*m_read_from_misc)(1) || m_read_from_misc->value() != WALLET_FILE_IDENTIFIER)throw MyException(sz_unrecognizable_wallet_file_format); + if (!(*m_read_from_misc)(2) || m_read_from_misc->value() != WALLET_FILE_SCHEMA_VERSION_0_0 || !(*m_read_from_misc)(4))throw MyException(sz_unrecognized_wallet_schema); + m_MasterSecret= *(m_read_from_misc->value()); ILogMessage(std::format("\t\tmaster secret: #{}", ro::base58(m_MasterSecret).operator const char* ()).c_str()); if(!m_MasterSecret.valid()) throw MyException(sz_cold_wallets_not_yet_implemented, __LINE__, __func__, SrcFilename); auto sizer = new wxBoxSizer(wxHORIZONTAL); @@ -26,11 +29,9 @@ display_wallet::display_wallet(wxWindow* parent, wxFileName& walletfile) : sizer->Add(m_rSizer, 50, wxGROW, 4); SetSizer(sizer); try { - ro::sql read_names_and_keys(m_db, - R"|(SELECT Names.name AS name, Keys.pubkey AS pubkey FROM Names INNER JOIN Keys ON Names.ROWID=Keys.id AND Keys.use=1 ORDER BY LOWER(name), name COLLATE BINARY;)|"); - while (read_names_and_keys.step() == Icompiled_sql::ROW) { - std::string name = read_names_and_keys.column(0); - auto pubkey = *read_names_and_keys.column(1); + while (m_read_names_and_keys->step() == Icompiled_sql::ROW) { + std::string name = m_read_names_and_keys->column(0); + auto pubkey = *(m_read_names_and_keys->column(1)); if (m_MasterSecret(name).timesBase() != pubkey)throw MyException(std::string(sz_public_key_of) + name + sz_fails_to_correspond); m_lSizer->Add( new wxStaticText( diff --git a/src/display_wallet.h b/src/display_wallet.h index b597c79..00243fa 100644 --- a/src/display_wallet.h +++ b/src/display_wallet.h @@ -7,6 +7,8 @@ public: private: typedef MenuLink MenuLink; std::unique_ptr m_db; + std::unique_ptr m_read_from_misc; + std::unique_ptr m_read_names_and_keys; ristretto255::CMasterSecret m_MasterSecret; wxBoxSizer* m_lSizer; wxBoxSizer* m_rSizer; diff --git a/src/serialization.h b/src/serialization.h index c35a143..cf6a371 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -75,7 +75,7 @@ l = std::size(a); pt = &a[0]; } - return std::span(static_cast(pt), l); + return std::span(static_cast(pt), l); } // Compile time test to see if a type has a blob array member From bc3f2c9dafe765c1d4fec454e26f1e82e885fc4b Mon Sep 17 00:00:00 2001 From: Cheng Date: Sat, 30 Sep 2023 05:13:25 +0000 Subject: [PATCH 18/20] Removed the old "is_serializable" in favor of C++20 concepts syntax. Changed the name to "has_machine_independent_representation" for a more intellible error explanation --- src/db_accessors.h | 2 +- src/ristretto255.h | 32 ++++++++++++++++--------------- src/serialization.h | 46 +++++++++++++++++++++------------------------ src/testbed.cpp | 2 +- src/unit_test.cpp | 2 +- 5 files changed, 41 insertions(+), 43 deletions(-) diff --git a/src/db_accessors.h b/src/db_accessors.h index 500c86f..a3f601b 100644 --- a/src/db_accessors.h +++ b/src/db_accessors.h @@ -107,7 +107,7 @@ namespace ro { template < typename T, typename std::enable_if::value, int >::type dummy_arg = 0 > void bind(int i, const T& j) { - static_assert(ro::serializable, "Don't know how to store this type in a database"); + static_assert(ro::has_machine_independent_representation, "Don't know how to store this type in a database"); (*this)->Isqlite3_bind(i, ro::serialize(j)); } typedef Icompiled_sql::sql_result result; diff --git a/src/ristretto255.h b/src/ristretto255.h index 3a4eb57..7faf593 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -61,7 +61,7 @@ namespace ristretto255 { ro::to_base64_string, ro::is_serializable, ro::serialize, ro::bin2hex, ro::hex2bin, ro::bin2hex, ro::CompileSizedString, - ro::serializable; + ro::has_machine_independent_representation; ; // a class representing ristretto255 elliptic points class point; @@ -119,7 +119,7 @@ namespace ristretto255 { assert(i == 0); } - template + template hsh& hashinto(const T& j, Args... args) { auto sj = ro::serialize(j); int i = crypto_generichash_blake2b_update( @@ -134,7 +134,7 @@ namespace ristretto255 { return *this; } - template hsh& operator << (const T& j) { + template hsh& operator << (const T& j) { auto sj = ro::serialize(j); auto i = crypto_generichash_blake2b_update( &(this->st), @@ -147,7 +147,7 @@ namespace ristretto255 { }; - static_assert(!serializable >, "Don't want to partially hash partial hashes"); + static_assert(!has_machine_independent_representation >, "Don't want to partially hash partial hashes"); // This constructs a finalized hash. // If it has one argument, and that argument is hsh (unfinalized hash) object, @@ -194,11 +194,12 @@ namespace ristretto255 { assert(i == 0); if (i) throw HashReuseException(); } - template< serializable T, typename... Args>explicit hash(const T& first, Args... args) { + template< has_machine_independent_representation T, typename... Args>explicit hash(const T& first, Args... args) { + hsh in; in << first; if constexpr (sizeof...(args) > 0) { - in.hashinto(args...); + in.hashinto( args...); } int i = crypto_generichash_blake2b_final( &in.st, @@ -280,6 +281,7 @@ namespace ristretto255 { "we need 512 bit randoms to ensure our points and scalars are uniformly distributed" ); // There should be scalar from hash, not point from hash + // libsodium already supplies a random point int i{ crypto_core_ristretto255_from_hash(&blob[0], &(x.blob)[0]) }; assert(i == 0); @@ -434,21 +436,21 @@ namespace ristretto255 { static_assert(ro::is_blob_field_type::value); static_assert(ro::is_blob_field_type >::value); static_assert(false == ro::is_blob_field_type::value); - static_assert(ro::is_serializable::value); - static_assert(ro::is_serializable&>::value); + static_assert(ro::has_machine_independent_representation); + static_assert(ro::has_machine_independent_representation&>); static_assert(ro::is_blob_field_type::value == false); - static_assert(ro::is_serializable::value); - static_assert(ro::is_serializable::value); - static_assert(ro::is_serializable::value == false); //false because uint8_t * has no inband terminator - static_assert(false == ro::is_serializable::value && !ro::is_constructible_from_v, wxString>, "wxStrings are apt to convert anything to anything, with surprising and unexpected results"); - static_assert(ro::is_serializable().ToUTF8())>::value == true); + static_assert(ro::has_machine_independent_representation); + static_assert(ro::has_machine_independent_representation); + static_assert(ro::has_machine_independent_representation == false); //false because uint8_t * has no inband terminator + static_assert(false == ro::has_machine_independent_representation && !ro::is_constructible_from_v, wxString>, "wxStrings are apt to convert anything to anything, with surprising and unexpected results"); + static_assert(ro::has_machine_independent_representation().ToUTF8())> == true); static_assert(ro::is_constructible_from_all_of, std::array>); - static_assert(ro::is_constructible_from_all_of, char*, short, unsigned short, hash<512>, point, scalar>, "want to be able to hash anything serializable"); + static_assert(ro::is_constructible_from_all_of, char*, short, unsigned short, hash<512>, point, scalar>, "want to be able to hash anything has_machine_independent_representation"); static_assert(false == ro::is_constructible_from_any_of, hash<256>>); static_assert(false == ro::is_constructible_from_any_of , byte*>, "do not want indiscriminate casts"); static_assert(false == ro::is_constructible_from_any_of , byte*>, "do not want indiscriminate casts "); static_assert(false == ro::is_constructible_from_v, float>); - static_assert(ro::is_serializable::value == false);//Need to convert floats to + static_assert(ro::has_machine_independent_representation == false);//Need to convert floats to // their machine independent representation, possibly through idexp, frexp // and DBL_MANT_DIG static_assert(sizeof(decltype(ro::serialize(std::declval())[0])) == 1); diff --git a/src/serialization.h b/src/serialization.h index cf6a371..0fcae03 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -59,12 +59,14 @@ // This template generates a span over an indexable byte type, // such as a C array or an std::array, but not pointers - template < typename T> - std::enable_if_t< - !std::is_pointer::value && - sizeof(std::declval()[0]) == 1, - std::span - > serialize(const T& a) { + template + concept byte_spannable = requires (T a) { + std::size(a); + a[0]; + } && sizeof(std::declval()[0]) == 1; + + template + auto serialize(const T& a) { int l; const void* pt; if constexpr (std::is_same_v, std::string>) { @@ -301,27 +303,21 @@ format being 44 digits in a base 58 representation.*/ // a compile time test to check if an object class has a machine independent representation - template struct is_serializable{ - template - static constexpr decltype(ro::serialize(std::declval()), bool()) test() { - if constexpr (sizeof...(Args2) > 0) { - return is_serializable::template test(); - } - else { - return true; - } - } - template static constexpr bool test(int = 0) { - return false; - } - static constexpr bool value = is_serializable::template test(); + template concept has_machine_independent_representation = requires (T a) { + serialize(a); }; - + + template static constexpr bool is_serializable() { + if constexpr (!has_machine_independent_representation) return false; + else if constexpr (sizeof...(Args) > 0) return is_serializable(); + else return true; + }; + template - concept serializable = is_serializable::value; + concept allserializable = is_serializable(); - static_assert( !serializable - && serializable, char*, std::span>, + static_assert( !has_machine_independent_representation + && allserializable, char*, std::span>, "concepts needed"); template ro::CompileSizedString< (2 * sizeof(T))>bin2hex(const T& pt) { @@ -351,4 +347,4 @@ return sz; } -} //End ro namespace \ No newline at end of file +} //End ro namespace diff --git a/src/testbed.cpp b/src/testbed.cpp index 1a44eac..a0ab14e 100644 --- a/src/testbed.cpp +++ b/src/testbed.cpp @@ -28,7 +28,7 @@ namespace testbed { using /*ristretto255::hash, ristretto255::hsh, */ristretto255::scalar, ristretto255::point, ro::serialize, ro::bin2hex, ro::hex2bin, ro::bin2hex, ro::fasthash, ro::CompileSizedString, - ro::base58, ro::serializable; + ro::base58, ro::has_machine_independent_representation; static constexpr char SrcFilename[]{ "src/testbed.cpp" }; diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 47403d5..62ea8f5 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -238,7 +238,7 @@ static bool checkDataConversionsProduceExpected(void){ point pt_a{ scl_a.timesBase() }; std::string str_pt_a = &(base58(pt_a))[0]; assert(base58::bin(str_pt_a.c_str()) == pt_a); - hash<256> hash_b{ hash_a,str_hash_a,scl_a,str_sclr_a,pt_a,str_pt_a,33, 66ull }; + hash<256> hash_b{ hash_a, str_hash_a, scl_a, str_sclr_a, pt_a, str_pt_a, 33, 66ull }; auto str_b = base58(hash_b).operator std::string(); if (str_b != "7cTScjKyUtmbvc28BV3ok51szgrQmaBa2YE5HPBcukC9e" ) { From dbe030ba211486b3c144bdb12fe57e10377e5d0b Mon Sep 17 00:00:00 2001 From: Cheng Date: Sat, 30 Sep 2023 20:11:14 +0000 Subject: [PATCH 19/20] Went all in on concepts because of nicer error messages But did not apply the concept to the parameter pack, event though I applied no end of cleverness to generate a variant concept, because visual studio gives meaningless error messages when applying a variant concept to a parameter pack. This will probably improve in later or different compilers, but right now, this feature does not work --- src/ristretto255.h | 3 +-- src/serialization.h | 23 ++++++++++++----------- src/unit_test.cpp | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/ristretto255.h b/src/ristretto255.h index 7faf593..91b0e30 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -58,7 +58,7 @@ void randombytes_buf(std::span in); namespace ristretto255 { using - ro::to_base64_string, ro::is_serializable, + ro::to_base64_string, ro::serialize, ro::bin2hex, ro::hex2bin, ro::bin2hex, ro::CompileSizedString, ro::has_machine_independent_representation; @@ -195,7 +195,6 @@ namespace ristretto255 { if (i) throw HashReuseException(); } template< has_machine_independent_representation T, typename... Args>explicit hash(const T& first, Args... args) { - hsh in; in << first; if constexpr (sizeof...(args) > 0) { diff --git a/src/serialization.h b/src/serialization.h index 0fcae03..7c4c997 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -118,7 +118,8 @@ inline auto serialize(const decltype(std::declval().ToUTF8()) sz){ return serialize(static_cast(sz)); } - /* + /* Don't do this. Disaster ensues, + inline auto serialize(const wxString& wxstr) { return serialize(static_cast(wxstr.ToUTF8())); } @@ -303,21 +304,21 @@ format being 44 digits in a base 58 representation.*/ // a compile time test to check if an object class has a machine independent representation - template concept has_machine_independent_representation = requires (T a) { - serialize(a); - }; - - template static constexpr bool is_serializable() { - if constexpr (!has_machine_independent_representation) return false; - else if constexpr (sizeof...(Args) > 0) return is_serializable(); - else return true; + template static constexpr bool serializable() { + if constexpr (requires(T a) { + serialize(a); + }) { + if constexpr (sizeof...(Args) > 0) return serializable(); + else return true; + } + else return false; }; template - concept allserializable = is_serializable(); + concept has_machine_independent_representation = serializable(); static_assert( !has_machine_independent_representation - && allserializable, char*, std::span>, + && has_machine_independent_representation, char*, std::span>, "concepts needed"); template ro::CompileSizedString< (2 * sizeof(T))>bin2hex(const T& pt) { diff --git a/src/unit_test.cpp b/src/unit_test.cpp index 62ea8f5..85750b2 100644 --- a/src/unit_test.cpp +++ b/src/unit_test.cpp @@ -238,7 +238,7 @@ static bool checkDataConversionsProduceExpected(void){ point pt_a{ scl_a.timesBase() }; std::string str_pt_a = &(base58(pt_a))[0]; assert(base58::bin(str_pt_a.c_str()) == pt_a); - hash<256> hash_b{ hash_a, str_hash_a, scl_a, str_sclr_a, pt_a, str_pt_a, 33, 66ull }; + hash<256> hash_b( hash_a, str_hash_a, scl_a, str_sclr_a, pt_a, wxString(str_pt_a).ToUTF8(), 33, 66ull); auto str_b = base58(hash_b).operator std::string(); if (str_b != "7cTScjKyUtmbvc28BV3ok51szgrQmaBa2YE5HPBcukC9e" ) { From 7ba674c29a1996813787bd8e5e71fffece7b0b1a Mon Sep 17 00:00:00 2001 From: Cheng Date: Mon, 2 Oct 2023 01:49:03 +0000 Subject: [PATCH 20/20] Cajoled visual studio into issuing intelligent error messages when an unserializable type is in the parameter pack. --- src/ristretto255.h | 4 +++- src/serialization.h | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/ristretto255.h b/src/ristretto255.h index 91b0e30..30cc919 100644 --- a/src/ristretto255.h +++ b/src/ristretto255.h @@ -195,10 +195,12 @@ namespace ristretto255 { if (i) throw HashReuseException(); } template< has_machine_independent_representation T, typename... Args>explicit hash(const T& first, Args... args) { + // not restraining the variant args by concept, because they get caught deeper in, + // and visual studio reporting of variant concept errors sucks. hsh in; in << first; if constexpr (sizeof...(args) > 0) { - in.hashinto( args...); + in.hashinto( ro::trigger_error(args)...); } int i = crypto_generichash_blake2b_final( &in.st, diff --git a/src/serialization.h b/src/serialization.h index 7c4c997..0eaf4ca 100644 --- a/src/serialization.h +++ b/src/serialization.h @@ -317,6 +317,9 @@ template concept has_machine_independent_representation = serializable(); + template + T trigger_error(T x) { return x; }; + static_assert( !has_machine_independent_representation && has_machine_independent_representation, char*, std::span>, "concepts needed");