From addb7fbe2dab0f93f5e358f120ceccd2f00b0c05 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 09:43:13 -0600 Subject: [PATCH 01/21] Code generators for efficient sorting algorithms usable on and off chain with off-chain standalone tests --- program/src/oracle/sort/clean | 3 + program/src/oracle/sort/run_tests | 18 ++ .../src/oracle/sort/sort_stable_base_gen.c | 92 +++++++ program/src/oracle/sort/test_sort_stable.c | 83 +++++++ program/src/oracle/sort/tmpl/sort_stable.c | 230 ++++++++++++++++++ .../src/oracle/sort/tmpl/sort_stable_base.c | 18 ++ 6 files changed, 444 insertions(+) create mode 100755 program/src/oracle/sort/clean create mode 100755 program/src/oracle/sort/run_tests create mode 100644 program/src/oracle/sort/sort_stable_base_gen.c create mode 100644 program/src/oracle/sort/test_sort_stable.c create mode 100644 program/src/oracle/sort/tmpl/sort_stable.c create mode 100644 program/src/oracle/sort/tmpl/sort_stable_base.c diff --git a/program/src/oracle/sort/clean b/program/src/oracle/sort/clean new file mode 100755 index 000000000..860ef1641 --- /dev/null +++ b/program/src/oracle/sort/clean @@ -0,0 +1,3 @@ +#!/bin/sh +rm -rfv bin + diff --git a/program/src/oracle/sort/run_tests b/program/src/oracle/sort/run_tests new file mode 100755 index 000000000..0eb277605 --- /dev/null +++ b/program/src/oracle/sort/run_tests @@ -0,0 +1,18 @@ +#!/bin/sh + +module purge || exit 1 +module load gcc-9.3.0 || exit 1 + +./clean || exit 1 +mkdir -pv bin || exit 1 + +CC="gcc -g -Wall -Werror -Wextra -Wconversion -Wstrict-aliasing=2 -Wimplicit-fallthrough=2 -pedantic -D_XOPEN_SOURCE=600 -O2 -march=native -std=c17" + +set -x + +$CC test_sort_stable.c -o bin/test_sort_stable || exit 1 + +bin/test_sort_stable || exit 1 + +echo all tests passed + diff --git a/program/src/oracle/sort/sort_stable_base_gen.c b/program/src/oracle/sort/sort_stable_base_gen.c new file mode 100644 index 000000000..70ac61597 --- /dev/null +++ b/program/src/oracle/sort/sort_stable_base_gen.c @@ -0,0 +1,92 @@ +#include +#include + +void +sort_gen( int n ) { + +# if 0 + + /* In register variant (PSEUDO OPS ~ 9+4n+3n(n-1)) + Assumes switch ~ 3 PSEUDO OPS (LDA,LD,JMP) -> 3 switch statements / 9 pseudo ops + Assumes load ~ 2 PSEUDO OPS (LDA,LD) -> n loads / 2n pseudo ops + Assumes store ~ " + Assumes cswap ~ 6 PSEUDO OPS (CMP,MOV,TESTEQ,CMOV,TESTNEQ,CMOV) / 6*0.5*n*(n-1) pseudo ops */ + +//printf( "static inline key_t * /* returns (sorted) x */\n" ); +//printf( "sort_network_stable( key_t * x, /* indexed [0,n) */\n" ); +//printf( " ulong n ) { /* assumes n in [0,%i) */\n", n ); + printf( " int c;\n" ); + printf( " key_t t" ); + for( int i=0; i 3 pseudo ops + Assumes cswap ~ 9 PSEUDO OPS (LDA,LDA,LD,LD,CMP,CMOV,CMOV,ST,ST) / 9*0.5*n*(n-1) pseudo ops */ + +//printf( "static inline key_t * /* returns (sorted) x */\n" ); +//printf( "sort_network_stable( key_t * x, /* indexed [0,n) */\n" ); +//printf( " ulong n ) { /* assumes n in [0,%i) */\n", n ); + + printf( " do { /* BEGIN AUTOGENERATED CODE (n=%2i) *****************************/\n", n ); + printf( "# define SORT_STABLE_CE(i,j) u = x[(SORT_IDX_T)i]; v = x[(SORT_IDX_T)j]; c = SORT_BEFORE( v, u ); x[(SORT_IDX_T)i] = c ? v : u; x[(SORT_IDX_T)j] = c ? u : v\n" ); + printf( " int c;\n" ); + printf( " SORT_KEY_T u;\n" ); + printf( " SORT_KEY_T v;\n" ); + printf( " switch( n ) {\n" ); + for( int i=n-1; i>=0; i-- ) { + printf( " case (SORT_IDX_T)%2i:", i+1 ); + for( int j=0; j +#include "../util/util.h" + +#define BEFORE(i,j) (((i)>>16)<((j)>>16)) + +#define SORT_NAME sort +#define SORT_KEY_T int +#define SORT_IDX_T int +#define SORT_BEFORE(i,j) BEFORE(i,j) +#include "tmpl/sort_stable.c" + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + +# define N 96 + int x[N]; + int y[N]; + int w[N]; + + /* Brute force validate small sizes via the 0-1 principle (with + additional information in the keys to validate stability as well). */ + + for( int n=0; n<=24; n++ ) { + printf( "Zero-One: Testing n=%i\n", n ); + for( long b=0L; b<(1L<>i) & 1L))<<16) | i; + for( int i=0; i=n || z[i]!=w[j] ) { printf( "FAIL (corrupt)\n" ); return 1; } + w[j] = -1; /* Mark that this entry has already been confirmed */ + } + for( int i=0; i=n || z[i]!=w[j] ) { printf( "FAIL (corrupt)\n" ); return 1; } + w[j] = -1; /* Mark that this entry has already been confirmed */ + } + for( int i=0; i /* For uint64_t */ +#include "../../util/align.h" /* For ALIGN_UP */ + +#ifndef SORT_NAME +#error "Define SORT_NAME" +#endif + +#ifndef SORT_KEY_T +#error "Define SORT_KEY_T; nominally a POD (plain-old-data) type" +#endif + +/* Define SORT_IDX_T to specify the data type used to index key arrays. + Default is uint64_t. */ + +#ifndef SORT_IDX_T +#define SORT_IDX_T uint64_t +#endif + +/* Define SORT_BEFORE to specify how sorted keys should be ordered. + Default is ascending as defined by the "<" operator for the type. + SORT_BEFORE(u,v) should be non-zero if key u should go strictly + before key v and zero otherwise. */ + +#ifndef SORT_BEFORE +#define SORT_BEFORE(u,v) ((u)<(v)) +#endif + +/* Define SORT_STATIC to specify the type of linkage the non-inlined + APIs should have (e.g. if defined to nothing, these will have + external linkage). Default is static linkage. */ + +#ifndef SORT_STATIC +#define SORT_STATIC static +#endif + +/* Define SORT_STATIC_INLINE to specify the type of linkage inlined + APIs should have (e.g. if defined to nothing, these will have + non-inlined external linkage). Default is static inline linkage. */ + +#ifndef SORT_STATIC_INLINE +#define SORT_STATIC_INLINE static inline +#endif + +/* Some macro preprocessor helpers */ + +#define SORT_C3(a,b,c)a##b##c +#define SORT_XC3(a,b,c)SORT_C3(a,b,c) +#define SORT_IMPL(impl)SORT_XC3(SORT_NAME,_,impl) + +SORT_STATIC_INLINE int +SORT_IMPL(stable_cnt_valid)( SORT_IDX_T cnt ) { + /* Written this way for complier warning free signed SORT_IDX_T and/or + byte size SORT_KEY_T support (e.g. compiler often will warn to the + effect "n>=0 always true" if idx is an unsigned type or + "n<=UINT64_MAX always true" if key is a byte type). */ + static uint64_t const max = ((UINT64_MAX - (uint64_t)alignof(SORT_KEY_T) + (uint64_t)1) / (uint64_t)sizeof(SORT_KEY_T)); + return !cnt || (((SORT_IDX_T)0)> 1; + SORT_KEY_T * yl = SORT_IMPL(stable_node)( xl,nl, tl ); + + SORT_KEY_T * xr = x + nl; + SORT_KEY_T * tr = t + nl; + SORT_IDX_T nr = n - nl; + SORT_KEY_T * yr = SORT_IMPL(stable_node)( xr,nr, tr ); + + /* If left subsort result ended up in orig array, merge into temp + array. Otherwise, merge into orig array. */ + + if( yl==xl ) x = t; + + /* At this point, note that yl does not overlap with the location for + merge output at this point. yr might overlap (with the right half) + with the location for merge output but this will still work in that + case. */ + + SORT_IDX_T i = (SORT_IDX_T)0; + SORT_IDX_T j = (SORT_IDX_T)0; + SORT_IDX_T k = (SORT_IDX_T)0; + + /* Note that nl and nr are both at least one at this point so at least + one interation of the loop body is necessary. */ + + do { + if( SORT_BEFORE( yr[k], yl[j] ) ) x[i++] = yr[k++]; + else x[i++] = yl[j++]; + } while( (j=nr ) break; } + else { x[i++] = yj; j++; if( j>=nl ) break; } + } + + for(;;) { /* Minimal exit condition tests */ + if( SORT_BEFORE( yr[k], yl[j] ) ) { x[i++] = yr[k]; k++; if( k>=nr ) break; } + else { x[i++] = yl[j]; j++; if( j>=nl ) break; } + } +# endif + + /* Append any stragglers (exactly one of these loops will execute) */ + + while( j Date: Tue, 11 Jan 2022 09:44:57 -0600 Subject: [PATCH 02/21] New pricing model usable on and off chain with off-chain standalone tests --- program/src/oracle/model/clean | 3 + program/src/oracle/model/model.h | 101 +++++++++++++++++ program/src/oracle/model/price_model.c | 113 ++++++++++++++++++++ program/src/oracle/model/run_tests | 18 ++++ program/src/oracle/model/test_price_model.c | 69 ++++++++++++ 5 files changed, 304 insertions(+) create mode 100755 program/src/oracle/model/clean create mode 100644 program/src/oracle/model/model.h create mode 100644 program/src/oracle/model/price_model.c create mode 100755 program/src/oracle/model/run_tests create mode 100644 program/src/oracle/model/test_price_model.c diff --git a/program/src/oracle/model/clean b/program/src/oracle/model/clean new file mode 100755 index 000000000..860ef1641 --- /dev/null +++ b/program/src/oracle/model/clean @@ -0,0 +1,3 @@ +#!/bin/sh +rm -rfv bin + diff --git a/program/src/oracle/model/model.h b/program/src/oracle/model/model.h new file mode 100644 index 000000000..0065f80dc --- /dev/null +++ b/program/src/oracle/model/model.h @@ -0,0 +1,101 @@ +#ifndef _pyth_oracle_model_model_h_ +#define _pyth_oracle_model_model_h_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Returns the minimum and maximum number of quotes the implementation + can handle */ + +static inline uint64_t +price_model_quote_min( void ) { + return (uint64_t)1; +} + +static inline uint64_t +price_model_quote_max( void ) { + return (UINT64_MAX-(uint64_t)alignof(int64_t)+(uint64_t)1) / (uint64_t)sizeof(int64_t); +} + +/* price_model_cnt_valid returns non-zero if cnt is a valid value or + zero if not. */ + +static inline int +price_model_cnt_valid( uint64_t cnt ) { + return price_model_quote_min()<=cnt && cnt<=price_model_quote_max(); +} + +/* price_model_scratch_footprint returns the number of bytes of scratch + space needed for an arbitrarily aligned scratch region required by + price_model to handle price_model_quote_min() to cnt quotes + inclusive. */ + +static inline uint64_t +price_model_scratch_footprint( uint64_t cnt ) { /* Assumes price_model_cnt_valid( cnt ) is true */ + /* cnt int64_t's plus worst case alignment padding, no overflow + possible as cnt is valid at this point */ + return cnt*(uint64_t)sizeof(int64_t)+(uint64_t)alignof(int64_t)-(uint64_t)1; +} + +/* price_model_core minimizes (to quote precision in a floor / round + toward negative infinity sense) the loss model of the given quotes. + Assumes valid inputs (e.g. cnt is at least 1 and not unreasonably + large ... typically a multiple of 3 but this is not required, + quote[i] for i in [0,cnt) are the quotes of interest on input, p25, + p50, p75 point to where to write model outputs, scratch points to a + suitable footprint srcatch region). + + Returns a pointer to the quotes sorted in ascending order. As such, + the min and max and any other rank statistic can be extracted easily + on return. This location will either be quote itself or to a + location in scratch. Use price_model below for a variant that always + replaces quote with the sorted quotes (potentially has extra ops for + copying). Further, on return, *_p25, *_p50, *_p75 will hold the loss + model minimizing values for the input quotes and the scratch region + was clobbered. + + Scratch points to a memory region of arbitrary alignment with at + least price_model_scratch_footprint( cnt ) bytes and it will be + clobbered on output. It is sufficient to use a normally aligned / + normally allocated / normally declared array of cnt int64_t's. + + The cost of this function is a fast and low variance (but not + completely data oblivious) O(cnt lg cnt) in the best / average / + worst cases. This function uses no heap / dynamic memory allocation. + It is thread safe provided it passed non-conflicting quote, output + and scratch arrays. It has a bounded call depth ~lg cnt <= ~64 (this + could reducd to O(1) by using a non-recursive sort/select + implementation under the hood if desired). */ + +int64_t * /* Returns pointer to sorted quotes (either quote or ALIGN_UP(scratch,int64_t)) */ +price_model_core( uint64_t cnt, /* Assumes price_model_cnt_valid( cnt ) is true */ + int64_t * restrict quote, /* Assumes quote[i] for i in [0,cnt) is the i-th quote on input */ + int64_t * restrict _p25, /* Assumes *_p25 is safe to write to the p25 model output */ + int64_t * restrict _p50, /* Assumes *_p50 " */ + int64_t * restrict _p75, /* Assumes *_p75 " */ + void * scratch ); /* Assumes a suitable scratch region */ + +/* Same as the above but always returns quote and quote always holds the + sorted quotes on return. */ + +static inline int64_t * +price_model( uint64_t cnt, + int64_t * restrict quote, + int64_t * restrict _p25, + int64_t * restrict _p50, + int64_t * restrict _p75, + void * scratch ) { + int64_t * restrict tmp = price_model_core( cnt, quote, _p25, _p50, _p75, scratch ); + if( tmp!=quote ) for( uint64_t idx=(uint64_t)0; idx>2. The current preference is not to do this as, though + this has stronger bum quote robustness, it results in p25==p50==p75 + when cnt==3. (In this case, the above wants to do an interpolation + between quotes 0 and 1 to for the p25 and between quotes 1 and 2 + for the p75. But limiting to just the inside quote results in + p25/p50/p75 all using the median quote.) + + A tweak to this option, for p25, is to use floor(cnt/4) == cnt>>2. + This is simple, has the same asymptotic behavior for large cnt, has + good behavior in the cnt==3 case and practically as good bum quote + rejection in the moderate cnt case. */ + + uint64_t p25_idx = cnt >> 2; + + *_p25 = sort_quote[p25_idx]; + + /* Extract the p50 */ + + if( (cnt & (uint64_t)1) ) { /* Odd number of quotes */ + + uint64_t p50_idx = cnt >> 1; /* ==ceil((cnt-1)/2) */ + + *_p50 = sort_quote[p50_idx]; + + } else { /* Even number of quotes (at least 2) */ + + uint64_t p50_idx_right = cnt >> 1; /* == ceil((cnt-1)/2)> 0 */ + uint64_t p50_idx_left = p50_idx_right - (uint64_t)1; /* ==floor((cnt-1)/2)>=0 (no overflow/underflow) */ + + int64_t vl = sort_quote[p50_idx_left ]; + int64_t vr = sort_quote[p50_idx_right]; + + /* Compute the average of vl and vr (with floor / round toward + negative infinity rounding and without possibility of + intermediate overflow). */ + + *_p50 = avg_2_int64( vl, vr ); + } + + /* Extract the p75 (this is the mirror image of the p25 case) */ + + uint64_t p75_idx = cnt - ((uint64_t)1) - p25_idx; + + *_p75 = sort_quote[p75_idx]; + + return sort_quote; +} + diff --git a/program/src/oracle/model/run_tests b/program/src/oracle/model/run_tests new file mode 100755 index 000000000..11c8f8731 --- /dev/null +++ b/program/src/oracle/model/run_tests @@ -0,0 +1,18 @@ +#!/bin/sh + +module purge || exit 1 +module load gcc-9.3.0 || exit 1 + +./clean || exit 1 +mkdir -pv bin || exit 1 + +CC="gcc -g -Wall -Werror -Wextra -Wconversion -Wstrict-aliasing=2 -Wimplicit-fallthrough=2 -pedantic -D_XOPEN_SOURCE=600 -O2 -march=native -std=c17" + +set -x + +$CC test_price_model.c price_model.c -o bin/test_price_model || exit 1 + +bin/test_price_model || exit 1 + +echo all tests passed + diff --git a/program/src/oracle/model/test_price_model.c b/program/src/oracle/model/test_price_model.c new file mode 100644 index 000000000..d5a7d6d95 --- /dev/null +++ b/program/src/oracle/model/test_price_model.c @@ -0,0 +1,69 @@ +#include +#include +#include +#include "../util/util.h" +#include "model.h" + +int +qcmp( void const * _p, + void const * _q ) { + int64_t p = *(int64_t const *)_p; + int64_t q = *(int64_t const *)_q; + if( p < q ) return -1; + if( p > q ) return 1; + return 0; +} + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + prng_t _prng[1]; + prng_t * prng = prng_join( prng_new( _prng, (uint32_t)0, (uint64_t)0 ) ); + +# define N 96 + + int64_t quote0 [N]; + int64_t quote [N]; + int64_t val [3]; + int64_t scratch[N]; + + int ctr = 0; + for( int iter=0; iter<10000000; iter++ ) { + if( !ctr ) { printf( "Completed %u iterations\n", iter ); ctr = 100000; } + ctr--; + + /* Generate a random test */ + + uint64_t cnt = (uint64_t)1 + (uint64_t)(prng_uint32( prng ) % (uint32_t)N); /* In [1,N], approx uniform IID */ + for( uint64_t idx=(uint64_t)0; idx>2; + uint64_t p50_idx = cnt>>1; + uint64_t p75_idx = cnt - (uint64_t)1 - p25_idx; + uint64_t is_even = (uint64_t)!(cnt & (uint64_t)1); + + if( val[0]!=quote[ p25_idx ] ) { printf( "FAIL (p25)\n" ); return 1; } + if( val[1]!=avg_2_int64( quote[ p50_idx-is_even ], quote[ p50_idx ] ) ) { printf( "FAIL (p50)\n" ); return 1; } + if( val[2]!=quote[ p75_idx ] ) { printf( "FAIL (p75)\n" ); return 1; } + } + +# undef N + + prng_delete( prng_leave( prng ) ); + + printf( "pass\n" ); + return 0; +} + From ef0bbd7563df9037c20acf7cc410ff7fb4127030 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 09:47:59 -0600 Subject: [PATCH 03/21] Draft hookup of the new pricing model --- program/src/oracle/sort.c | 82 ------------------------- program/src/oracle/sort.h | 18 ------ program/src/oracle/test_oracle.c | 1 - program/src/oracle/upd_aggregate.h | 99 +++++++++++++++++++++--------- 4 files changed, 70 insertions(+), 130 deletions(-) delete mode 100644 program/src/oracle/sort.c delete mode 100644 program/src/oracle/sort.h diff --git a/program/src/oracle/sort.c b/program/src/oracle/sort.c deleted file mode 100644 index 4cb53ebc3..000000000 --- a/program/src/oracle/sort.c +++ /dev/null @@ -1,82 +0,0 @@ -#include "sort.h" - -#ifndef __bpf__ -#include -#endif - -static inline void swap_int64( int64_t *a, int64_t *b ) -{ - int64_t const temp = *a; - *a = *b; - *b = temp; -} - -static inline int partition_int64( int64_t* v, int i, int j ) -{ -#ifndef __bpf__ -#ifndef NDEBUG - int const i0 = i; - int const j0 = j; -#endif -#endif - - int const p = i; - int64_t const pv = v[ p ]; - - while ( i < j ) { - while ( v[ i ] <= pv && i <= j ) { - ++i; - } - while ( v[ j ] > pv ) { - --j; - } - if ( i < j ) { - swap_int64( &v[ i ], &v[ j ] ); - } - } - swap_int64( &v[ p ], &v[ j ] ); - -#ifndef __bpf__ -#ifndef NDEBUG - int k; - for ( k = i0; k < j; ++k ) { - assert( v[ k ] <= pv ); - } - assert( v[ j ] == pv ); - for ( k = j + 1; k <= j0; ++k ) { - assert( v[ k ] > pv ); - } -#endif -#endif - - return j; -} - -static void qsort_help_int64( int64_t* const v, int i, int j ) -{ - if ( i >= j ) { - return; - } - int p = partition_int64( v, i, j ); - qsort_help_int64( v, i, p - 1 ); - qsort_help_int64( v, p + 1, j ); -} - -void qsort_int64( int64_t* const v, unsigned const size ) -{ - qsort_help_int64( v, 0, ( int )size - 1 ); - -#ifndef __bpf__ -#ifndef NDEBUG - if ( size ) { - int64_t x = v[ 0 ]; - unsigned i = 1; - for ( ; i < size; ++i ) { - assert( x <= v[ i ] ); - x = v[ i ]; - } - } -#endif -#endif -} - diff --git a/program/src/oracle/sort.h b/program/src/oracle/sort.h deleted file mode 100644 index ebf67198c..000000000 --- a/program/src/oracle/sort.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#ifdef __bpf__ -#include -#else -#include -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -void qsort_int64( int64_t* v, unsigned size ); - -#ifdef __cplusplus -} -#endif - diff --git a/program/src/oracle/test_oracle.c b/program/src/oracle/test_oracle.c index 5924b613b..bdda372ba 100644 --- a/program/src/oracle/test_oracle.c +++ b/program/src/oracle/test_oracle.c @@ -1,7 +1,6 @@ char heap_start[8192]; #define PC_HEAP_START (heap_start) #include "oracle.c" -#include "sort.c" #include uint64_t MAPPING_ACCOUNT_LAMPORTS = 143821440; diff --git a/program/src/oracle/upd_aggregate.h b/program/src/oracle/upd_aggregate.h index 1aa5f8e5e..b2c01b06d 100644 --- a/program/src/oracle/upd_aggregate.h +++ b/program/src/oracle/upd_aggregate.h @@ -1,8 +1,8 @@ #pragma once #include "oracle.h" +#include "model/model.h" #include "pd.h" -#include "sort.h" #include @@ -201,39 +201,76 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest ptr->agg_.pub_slot_ = slot; // publish slot-time of agg. price ptr->timestamp_ = timestamp; + // identify valid quotes + // compute the aggregate prices and ranges uint32_t numv = 0; uint32_t vidx[ PC_COMP_SIZE ]; - // identify valid quotes and order them by price - for ( uint32_t i = 0; i != ptr->num_; ++i ) { - pc_price_comp_t *iptr = &ptr->comp_[i]; - // copy contributing price to aggregate snapshot - iptr->agg_ = iptr->latest_; - // add quote to sorted permutation array if it is valid - int64_t slot_diff = ( int64_t )slot - ( int64_t )( iptr->agg_.pub_slot_ ); - if ( iptr->agg_.status_ == PC_STATUS_TRADING && - iptr->agg_.conf_ != 0UL && - iptr->agg_.price_ > 0L && - slot_diff >= 0 && slot_diff <= PC_MAX_SEND_LATENCY ) { - vidx[numv++] = i; + int64_t agg_price; + int64_t agg_conf; + { + uint32_t nprcs = (uint32_t)0; + int64_t prcs[ PC_COMP_SIZE * 3 ]; // ~0.75KiB for current PC_COMP_SIZE (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + for ( uint32_t i = 0; i != ptr->num_; ++i ) { + pc_price_comp_t *iptr = &ptr->comp_[i]; + // copy contributing price to aggregate snapshot + iptr->agg_ = iptr->latest_; + // add quote to sorted permutation array if it is valid + int64_t slot_diff = ( int64_t )slot - ( int64_t )( iptr->agg_.pub_slot_ ); + int64_t price = iptr->agg_.price_; + int64_t conf = ( int64_t )( iptr->agg_.conf_ ); + if ( iptr->agg_.status_ == PC_STATUS_TRADING && + (int64_t)0 < conf && conf < price && conf <= (INT64_MAX-price) && // No overflow for INT64_MAX-price as price>0 + slot_diff >= 0 && slot_diff <= PC_MAX_SEND_LATENCY ) { + vidx[numv++] = i; // FIXME: GRAFT FOR OLD PRICE MODEL + prcs[ nprcs++ ] = price - conf; // No overflow as 0 < conf < price + prcs[ nprcs++ ] = price; + prcs[ nprcs++ ] = price + conf; // No overflow as 0 < conf <= INT64_MAX-price + } + } + + // too few valid quotes + ptr->num_qt_ = numv; // FIXME: TEMPORARY GRAFT (ALL RETURN PATHS SET NUM_QT TO SOMETHING SENSIBLE) + if ( numv == 0 || numv < ptr->min_pub_ ) { + ptr->agg_.status_ = PC_STATUS_UNKNOWN; + return; } - } - uint32_t nprcs = 0; - int64_t prcs[ PC_COMP_SIZE * 2 ]; - for ( uint32_t i = 0; i < numv; ++i ) { - pc_price_comp_t const *iptr = &ptr->comp_[ vidx[ i ] ]; - int64_t const price = iptr->agg_.price_; - int64_t const conf = ( int64_t )( iptr->agg_.conf_ ); - prcs[ nprcs++ ] = price - conf; - prcs[ nprcs++ ] = price + conf; + // evaluate the model to get the p25/p50/p75 prices + // note: numv>0 and nprcs = 3*numv at this point + int64_t agg_p25; + int64_t agg_p75; + int64_t scratch[ PC_COMP_SIZE * 3 ]; // ~0.75KiB for current PC_COMP_SIZE (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) + price_model_core( (uint64_t)nprcs, prcs, &agg_p25, &agg_price, &agg_p75, scratch ); + + // get the left and right confidences + // note that as valid quotes have positive prices currently and + // agg_p25, agg_price, agg_p75 are ordered, this calculation can't + // overflow / underflow + int64_t agg_conf_left = agg_price - agg_p25; + int64_t agg_conf_right = agg_p75 - agg_price; + + // use the larger of the left and right confidences + agg_conf = agg_conf_right > agg_conf_left ? agg_conf_right : agg_conf_left; + + // if the confidences end up at zero, we abort + // this is paranoia as it is currently not possible when nprcs>2 and + // positive confidences given the current pricing model + if( agg_conf <= (int64_t)0 ) { + ptr->agg_.status_ = PC_STATUS_UNKNOWN; + return; + } } - qsort_int64( prcs, nprcs ); + + // FIXME: MOST OF THE REST OF THE CODE HERE CAN PROBABLY BE REMOVED / + // CONDENSED AND/OR MADE MORE EFFICIENT. AND IT PROBABLY WILL + // REPLACED SOON BY THE VWAP MODEL UNDER DEVELOPMENT. IT IS KEPT HERE + // TO KEEP THE CURRENT VWAP CALCULATION AS CLOSE AS POSSIBLE TO WHAT + // HOW IT CURRENTLY WORKS. uint32_t numa = 0; uint32_t aidx[PC_COMP_SIZE]; - - if ( nprcs ) { - int64_t const mprc = ( prcs[ numv - 1 ] + prcs[ numv ] ) / 2; + { + int64_t const mprc = agg_price; // FIXME: GRAFT TO NEW CALC TO OLD CALC int64_t const lb = mprc / 5; int64_t const ub = ( mprc > LONG_MAX / 5 ) ? LONG_MAX : mprc * 5; @@ -243,7 +280,7 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest int64_t const prc = iptr->agg_.price_; if ( prc >= lb && prc <= ub ) { uint32_t j = numa++; - for( ; j > 0 && ptr->comp_[ aidx[ j - 1 ] ].agg_.price_ > prc; --j ) { + for( ; j > 0 && ptr->comp_[ aidx[ j - 1 ] ].agg_.price_ > prc; --j ) { // FIXME: O(N^2) aidx[ j ] = aidx[ j - 1 ]; } aidx[ j ] = idx; @@ -268,6 +305,8 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest ptr->agg_.price_ = iptr->agg_.price_; ptr->agg_.conf_ = iptr->agg_.conf_; upd_twap( ptr, agg_diff, qs ); + ptr->agg_.price_ = agg_price; // FIXME: OVERRIDE OLD PRICE MODEL + ptr->agg_.conf_ = (uint64_t)agg_conf; // FIXME: OVERRIDE OLD PRICE MODEL return; } @@ -358,7 +397,7 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest // sort upper prices and weights for ( uint32_t i = 0; i < numa; ++i ) { uint32_t j = i; - for ( ; j > 0 && pd_lt( &qs->uprice_[ i ], &prices[ j - 1 ], qs->fact_ ); --j ) { + for ( ; j > 0 && pd_lt( &qs->uprice_[ i ], &prices[ j - 1 ], qs->fact_ ); --j ) { // FIXME: O(N^2) prices[ j ] = prices[ j - 1 ]; weights[ j ] = weights[ j - 1 ]; } @@ -369,7 +408,7 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest // sort lower prices and weights for ( uint32_t i = 0; i < numa; ++i ) { uint32_t j = i; - for ( ; j > 0 && pd_lt( &qs->lprice_[ i ], &prices[ j - 1 ], qs->fact_ ); --j ) { + for ( ; j > 0 && pd_lt( &qs->lprice_[ i ], &prices[ j - 1 ], qs->fact_ ); --j ) { // FIXME: O(N^2) prices[ j ] = prices[ j - 1 ]; weights[ j ] = weights[ j - 1 ]; } @@ -394,6 +433,8 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest pd_adjust( cptr, ptr->expo_, qs->fact_ ); ptr->agg_.conf_ = ( uint64_t )( cptr->v_ ); upd_twap( ptr, agg_diff, qs ); + ptr->agg_.price_ = agg_price; // FIXME: OVERRIDE OLD PRICE MODEL + ptr->agg_.conf_ = (uint64_t)agg_conf; // FIXME: OVERRIDE OLD PRICE MODEL } #ifdef __cplusplus From 8fbbc324e492f6dfd00bb026a5c990b0bebc1d19 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 11:14:46 -0600 Subject: [PATCH 04/21] Updated to reflect new aggregation logic --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3a24236c..7756bc60e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ set( PC_SRC pc/request.cpp; pc/rpc_client.cpp; pc/user.cpp; - program/src/oracle/sort.c + program/src/model/price_model.c ) set( PC_HDR From 5ca547f37f698e4fe0733bdbb76f9ee885c8f0f1 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 11:24:56 -0600 Subject: [PATCH 05/21] Docker doesn't understand the restrict keyword --- program/src/oracle/model/model.h | 28 +++++++++++++------------- program/src/oracle/model/price_model.c | 12 +++++------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/program/src/oracle/model/model.h b/program/src/oracle/model/model.h index 0065f80dc..0510b71ce 100644 --- a/program/src/oracle/model/model.h +++ b/program/src/oracle/model/model.h @@ -71,25 +71,25 @@ price_model_scratch_footprint( uint64_t cnt ) { /* Assumes price_model_cnt_valid could reducd to O(1) by using a non-recursive sort/select implementation under the hood if desired). */ -int64_t * /* Returns pointer to sorted quotes (either quote or ALIGN_UP(scratch,int64_t)) */ -price_model_core( uint64_t cnt, /* Assumes price_model_cnt_valid( cnt ) is true */ - int64_t * restrict quote, /* Assumes quote[i] for i in [0,cnt) is the i-th quote on input */ - int64_t * restrict _p25, /* Assumes *_p25 is safe to write to the p25 model output */ - int64_t * restrict _p50, /* Assumes *_p50 " */ - int64_t * restrict _p75, /* Assumes *_p75 " */ - void * scratch ); /* Assumes a suitable scratch region */ +int64_t * /* Returns pointer to sorted quotes (either quote or ALIGN_UP(scratch,int64_t)) */ +price_model_core( uint64_t cnt, /* Assumes price_model_cnt_valid( cnt ) is true */ + int64_t * quote, /* Assumes quote[i] for i in [0,cnt) is the i-th quote on input */ + int64_t * _p25, /* Assumes *_p25 is safe to write to the p25 model output */ + int64_t * _p50, /* Assumes *_p50 " */ + int64_t * _p75, /* Assumes *_p75 " */ + void * scratch ); /* Assumes a suitable scratch region */ /* Same as the above but always returns quote and quote always holds the sorted quotes on return. */ static inline int64_t * -price_model( uint64_t cnt, - int64_t * restrict quote, - int64_t * restrict _p25, - int64_t * restrict _p50, - int64_t * restrict _p75, - void * scratch ) { - int64_t * restrict tmp = price_model_core( cnt, quote, _p25, _p50, _p75, scratch ); +price_model( uint64_t cnt, + int64_t * quote, + int64_t * _p25, + int64_t * _p50, + int64_t * _p75, + void * scratch ) { + int64_t * tmp = price_model_core( cnt, quote, _p25, _p50, _p75, scratch ); if( tmp!=quote ) for( uint64_t idx=(uint64_t)0; idx Date: Tue, 11 Jan 2022 11:43:58 -0600 Subject: [PATCH 06/21] Solana uses its own custom replacement for stdint --- program/src/oracle/model/model.h | 2 +- program/src/oracle/sort/tmpl/sort_stable.c | 4 +- program/src/oracle/util/align.h | 57 +++++++++++++++++ program/src/oracle/util/hash.h | 74 ++++++++++++++++++++++ program/src/oracle/util/sar.h | 74 ++++++++++++++++++++++ 5 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 program/src/oracle/util/align.h create mode 100644 program/src/oracle/util/hash.h create mode 100644 program/src/oracle/util/sar.h diff --git a/program/src/oracle/model/model.h b/program/src/oracle/model/model.h index 0510b71ce..59cc9d8c0 100644 --- a/program/src/oracle/model/model.h +++ b/program/src/oracle/model/model.h @@ -1,7 +1,7 @@ #ifndef _pyth_oracle_model_model_h_ #define _pyth_oracle_model_model_h_ -#include +#include "../util/compat_stdint.h" #include #ifdef __cplusplus diff --git a/program/src/oracle/sort/tmpl/sort_stable.c b/program/src/oracle/sort/tmpl/sort_stable.c index 45cb68dc5..c8f260c53 100644 --- a/program/src/oracle/sort/tmpl/sort_stable.c +++ b/program/src/oracle/sort/tmpl/sort_stable.c @@ -57,8 +57,8 @@ Other defines exist to change the sort criteria / direction, linkage and so forth. See below for details. */ -#include /* For uint64_t */ -#include "../../util/align.h" /* For ALIGN_UP */ +#include "../../util/compat_stdint.h" /* For uint64_t */ +#include "../../util/align.h" /* For ALIGN_UP */ #ifndef SORT_NAME #error "Define SORT_NAME" diff --git a/program/src/oracle/util/align.h b/program/src/oracle/util/align.h new file mode 100644 index 000000000..023e85735 --- /dev/null +++ b/program/src/oracle/util/align.h @@ -0,0 +1,57 @@ +#ifndef _pyth_oracle_util_align_h_ +#define _pyth_oracle_util_align_h_ + +#include "compat_stdint.h" /* For uintptr_t */ +#include /* For alignof */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* align_ispow2 returns non-zero if a is a non-negative integral power + of 2 and zero otherwise. */ + +static inline int +align_ispow2( uintptr_t a ) { + uintptr_t mask = a - (uintptr_t)1; + return (a>(uintptr_t)0) & !(a & mask); +} + +/* align_isaligned returns non-zero if p has byte alignment a. Assumes + align_ispow2(a) is true. */ + +static inline int +align_isaligned( void * p, + uintptr_t a ) { + uintptr_t mask = a - (uintptr_t)1; + return !(((uintptr_t)p) & mask); +} + +/* align_dn/align_up aligns p to the first byte at or before / after p + with alignment a. Assumes align_ispow2(a) is true. */ + +static inline void * +align_dn( void * p, + uintptr_t a ) { + uintptr_t mask = a - (uintptr_t)1; + return (void *)(((uintptr_t)p) & (~mask)); +} + +static inline void * +align_up( void * p, + uintptr_t a ) { + uintptr_t mask = a - (uintptr_t)1; + return (void *)((((uintptr_t)p) + mask) & (~mask)); +} + +/* Helper macros for aligning pointers up or down to compatible + alignments for type T. FIXME: ADD UNIT TEST COVERAGE */ + +#define ALIGN_DN( p, T ) ((T *)align_dn( p, (uintptr_t)alignof(T) )) +#define ALIGN_UP( p, T ) ((T *)align_up( p, (uintptr_t)alignof(T) )) + +#ifdef __cplusplus +} +#endif + +#endif /* _pyth_oracle_util_align_h_ */ diff --git a/program/src/oracle/util/hash.h b/program/src/oracle/util/hash.h new file mode 100644 index 000000000..4bdd1023a --- /dev/null +++ b/program/src/oracle/util/hash.h @@ -0,0 +1,74 @@ +#ifndef _pyth_oracle_util_hash_h_ +#define _pyth_oracle_util_hash_h_ + +/* Useful hashing utilities */ + +#include "compat_stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* High quality (full avalanche) high speed integer to integer hashing. + hash_uint32 has the properties that [0,2^32) hashes to a random + looking permutation of [0,2^32) and hash(0)==0. Similarly for + hash_uint64. Based on Google's Murmur3 hash finalizer (public + domain). Not cryptographically secure but passes various strict + tests of randomness when used as a PRNG (see prng.h). */ + +static inline uint32_t +hash_uint32( uint32_t x ) { + x ^= x >> 16; + x *= UINT32_C( 0x85ebca6b ); + x ^= x >> 13; + x *= UINT32_C( 0xc2b2ae35 ); + x ^= x >> 16; + return x; +} + +static inline uint64_t +hash_uint64( uint64_t x ) { + x ^= x >> 33; + x *= UINT64_C( 0xff51afd7ed558ccd ); + x ^= x >> 33; + x *= UINT64_C( 0xc4ceb9fe1a85ec53 ); + x ^= x >> 33; + return x; +} + +/* Inverses of the above. E.g.: + hash_inverse_uint32( hash_uint32( (uint32_t)x ) )==(uint32_t)x + and: + hash_uint32( hash_inverse_uint32( (uint32_t)x ) )==(uint32_t)x + Similarly for hash_inverse_uint64. These by themselves are similar + quality hashes to the above and having the inverses of the above can + be useful. The fact these have (nearly) identical operations / + operation counts concretely demonstrates that none of these are + standalone cryptographically secure. */ + +static inline uint32_t +hash_inverse_uint32( uint32_t x ) { + x ^= x >> 16; + x *= UINT32_C( 0x7ed1b41d ); + x ^= (x >> 13) ^ (x >> 26); + x *= UINT32_C( 0xa5cb9243 ); + x ^= x >> 16; + return x; +} + +static inline uint64_t +hash_inverse_uint64( uint64_t x ) { + x ^= x >> 33; + x *= UINT64_C( 0x9cb4b2f8129337db ); + x ^= x >> 33; + x *= UINT64_C( 0x4f74430c22a54005 ); + x ^= x >> 33; + return x; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _pyth_oracle_util_hash_h_ */ + diff --git a/program/src/oracle/util/sar.h b/program/src/oracle/util/sar.h new file mode 100644 index 000000000..9b0722560 --- /dev/null +++ b/program/src/oracle/util/sar.h @@ -0,0 +1,74 @@ +#ifndef _pyth_oracle_util_sar_h_ +#define _pyth_oracle_util_sar_h_ + +/* Portable arithmetic shift right. */ + +#include "compat_stdint.h" + +/* Define PYTH_ORACLE_UTIL_SAR_USE_PLATFORM to non-zero if the platform + does arithmetic right shift signed integer and zero if the platform + does not or is not known to. PYTH_ORACLE_UTIL_SAR_USE_PLATFORM + defaults to zero. */ + +#ifndef PYTH_ORACLE_UTIL_SAR_USE_PLATFORM +#define PYTH_ORACLE_UTIL_SAR_USE_PLATFORM 0 +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +static inline int8_t +sar_int8( int8_t x, + int n ) { /* Assumed in [0,7] */ +# if PYTH_ORACLE_UTIL_SAR_USE_PLATFORM + return x >> n; +# else + uint8_t ux = (uint8_t)x; + uint8_t s = (uint8_t)-(ux >> 7); /* get the sign, ((int8_t)s)==-1 if x<0 and 0 otherwise */ + return (int8_t)(((ux ^ s) >> n) ^ s); +# endif +} + +static inline int16_t +sar_int16( int16_t x, + int n ) { /* Assumed in [0,15] */ +# if PYTH_ORACLE_UTIL_SAR_USE_PLATFORM + return x >> n; +# else + uint16_t ux = (uint16_t)x; + uint16_t s = (uint16_t)-(ux >> 15); /* get the sign, ((int16_t)s)==-1 if x<0 and 0 otherwise */ + return (int16_t)(((ux ^ s) >> n) ^ s); +# endif +} + +static inline int32_t +sar_int32( int32_t x, + int n ) { /* Assumed in [0,31] */ +# if PYTH_ORACLE_UTIL_SAR_USE_PLATFORM + return x >> n; +# else + uint32_t ux = (uint32_t)x; + uint32_t s = -(ux >> 31); /* get the sign, ((int32_t)s)==-1 if x<0 and 0 otherwise */ + return (int32_t)(((ux ^ s) >> n) ^ s); +# endif +} + +static inline int64_t +sar_int64( int64_t x, + int n ) { /* Assumed in [0,63] */ +# if PYTH_ORACLE_UTIL_SAR_USE_PLATFORM + return x >> n; +# else + uint64_t ux = (uint64_t)x; + uint64_t s = -(ux >> 63); /* get the sign, ((int64_t)s)==-1 if x<0 and 0 otherwise */ + return (int64_t)(((ux ^ s) >> n) ^ s); +# endif +} + +#ifdef __cplusplus +} +#endif + +#endif /* _pyth_oracle_util_sar_h_ */ + From a911ded09c9928767598652ba368c5a32fa6f13d Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 11:46:49 -0600 Subject: [PATCH 07/21] Solana uses its own custom replacement for stdint --- program/src/oracle/util/compat_stdint.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 program/src/oracle/util/compat_stdint.h diff --git a/program/src/oracle/util/compat_stdint.h b/program/src/oracle/util/compat_stdint.h new file mode 100644 index 000000000..9490e2cb6 --- /dev/null +++ b/program/src/oracle/util/compat_stdint.h @@ -0,0 +1,13 @@ +#ifndef _pyth_oracle_util_compat_stdint_h_ +#define _pyth_oracle_util_compat_stdint_h_ + +/* solana uses its own definitions for stdint types and that can cause + problems with compilation */ + +#ifdef __bpf__ +#include +#else +#include +#endif + +#endif /* _pyth_oracle_util_compat_stdint_h_ */ From 41fb60c57d92694ad4afbe08427dc2cdd130ed06 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 11:55:44 -0600 Subject: [PATCH 08/21] Hack to work around docker linkage limitation --- program/src/oracle/upd_aggregate.h | 1 + 1 file changed, 1 insertion(+) diff --git a/program/src/oracle/upd_aggregate.h b/program/src/oracle/upd_aggregate.h index b2c01b06d..3ccc86a17 100644 --- a/program/src/oracle/upd_aggregate.h +++ b/program/src/oracle/upd_aggregate.h @@ -2,6 +2,7 @@ #include "oracle.h" #include "model/model.h" +#include "model/price_model.c" /* FIXME: HACK TO DEAL WITH DOCKER LINKAGE ISSUES */ #include "pd.h" #include From 27f5ed9e34cb820148331aa45f34b24cae5b2234 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 12:06:47 -0600 Subject: [PATCH 09/21] Solana's replacement for stdint isn't complete --- program/src/oracle/util/compat_stdint.h | 1 + 1 file changed, 1 insertion(+) diff --git a/program/src/oracle/util/compat_stdint.h b/program/src/oracle/util/compat_stdint.h index 9490e2cb6..2c3988e20 100644 --- a/program/src/oracle/util/compat_stdint.h +++ b/program/src/oracle/util/compat_stdint.h @@ -6,6 +6,7 @@ #ifdef __bpf__ #include +typedef uint64_t uintptr_t; #else #include #endif From f9837336b3898bf1fd8c8495b70cc80e30a86d9b Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 12:56:36 -0600 Subject: [PATCH 10/21] Minor ops tweak --- program/src/oracle/sort/tmpl/sort_stable.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/program/src/oracle/sort/tmpl/sort_stable.c b/program/src/oracle/sort/tmpl/sort_stable.c index c8f260c53..9a6d0b8a5 100644 --- a/program/src/oracle/sort/tmpl/sort_stable.c +++ b/program/src/oracle/sort/tmpl/sort_stable.c @@ -202,10 +202,10 @@ SORT_IMPL(stable_node)( SORT_KEY_T * x, } # endif - /* Append any stragglers (exactly one of these loops will execute) */ + /* Append any stragglers */ - while( j Date: Tue, 11 Jan 2022 13:17:53 -0600 Subject: [PATCH 11/21] More minor op count tweaks --- program/src/oracle/sort/tmpl/sort_stable.c | 34 ++++++++++------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/program/src/oracle/sort/tmpl/sort_stable.c b/program/src/oracle/sort/tmpl/sort_stable.c index 9a6d0b8a5..9f95e5be7 100644 --- a/program/src/oracle/sort/tmpl/sort_stable.c +++ b/program/src/oracle/sort/tmpl/sort_stable.c @@ -163,13 +163,23 @@ SORT_IMPL(stable_node)( SORT_KEY_T * x, /* Note that nl and nr are both at least one at this point so at least one interation of the loop body is necessary. */ - do { + for(;;) { /* Minimal C language operations */ + if( SORT_BEFORE( yr[k], yl[j] ) ) { + x[i++] = yr[k++]; + if( k>=nr ) { /* append left stragglers (at least one) */ do x[i++] = yl[j++]; while( j=nl ) { /* append right stragglers (at least one) */ do x[i++] = yr[k++]; while( k=nr ) break; } - else { x[i++] = yj; j++; if( j>=nl ) break; } - } - - for(;;) { /* Minimal exit condition tests */ - if( SORT_BEFORE( yr[k], yl[j] ) ) { x[i++] = yr[k]; k++; if( k>=nr ) break; } - else { x[i++] = yl[j]; j++; if( j>=nl ) break; } - } -# endif - /* Append any stragglers */ if( j Date: Thu, 20 Jan 2022 12:21:48 -0600 Subject: [PATCH 12/21] Use STYLE macro semantics like other includes --- program/src/oracle/util/compat_stdint.h | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/program/src/oracle/util/compat_stdint.h b/program/src/oracle/util/compat_stdint.h index 2c3988e20..1842cc52c 100644 --- a/program/src/oracle/util/compat_stdint.h +++ b/program/src/oracle/util/compat_stdint.h @@ -1,14 +1,28 @@ #ifndef _pyth_oracle_util_compat_stdint_h_ #define _pyth_oracle_util_compat_stdint_h_ -/* solana uses its own definitions for stdint types and that can cause - problems with compilation */ +/* Include functionality from . Define + PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE to indicate how to do this: + 0 - Use stdint.h directly + 1 - Use solana_sdk.h (solana uses its own definitions for stdint + types and that can conflicts with stdint.h) + Defaults to 0 or 1 depending on if __bpf__ is set. */ -#ifdef __bpf__ +#ifndef PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE +#ifndef __bpf__ +#define PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE 0 +#else +#define PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE 1 +#endif +#endif + +#if PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE==0 +#include +#elif PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE==1 #include typedef uint64_t uintptr_t; #else -#include +#error "Unsupported PYTH_ORACLE_UTIL_COMPAT_STDINT_STYLE" #endif #endif /* _pyth_oracle_util_compat_stdint_h_ */ From bf0dfd554bc55c145558450a1a9294d6c781b347 Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Sat, 12 Feb 2022 11:12:22 -0600 Subject: [PATCH 13/21] Renamed to be consistent with other files in this module --- program/src/oracle/model/{model.h => price_model.h} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename program/src/oracle/model/{model.h => price_model.h} (100%) diff --git a/program/src/oracle/model/model.h b/program/src/oracle/model/price_model.h similarity index 100% rename from program/src/oracle/model/model.h rename to program/src/oracle/model/price_model.h From ba2259a8dfe816634ac555cf568d68a4dcc19c6d Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Sat, 12 Feb 2022 11:17:01 -0600 Subject: [PATCH 14/21] Updated for file name change --- program/src/oracle/model/price_model.c | 2 +- program/src/oracle/model/test_price_model.c | 2 +- program/src/oracle/upd_aggregate.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/program/src/oracle/model/price_model.c b/program/src/oracle/model/price_model.c index 9b83090ee..303d52e0a 100644 --- a/program/src/oracle/model/price_model.c +++ b/program/src/oracle/model/price_model.c @@ -1,4 +1,4 @@ -#include "model.h" +#include "price_model.h" #include "../util/avg.h" /* For avg_2_int64 */ #define SORT_NAME int64_sort_ascending diff --git a/program/src/oracle/model/test_price_model.c b/program/src/oracle/model/test_price_model.c index d5a7d6d95..321ff1746 100644 --- a/program/src/oracle/model/test_price_model.c +++ b/program/src/oracle/model/test_price_model.c @@ -2,7 +2,7 @@ #include #include #include "../util/util.h" -#include "model.h" +#include "price_model.h" int qcmp( void const * _p, diff --git a/program/src/oracle/upd_aggregate.h b/program/src/oracle/upd_aggregate.h index 3ccc86a17..cd0f79bf8 100644 --- a/program/src/oracle/upd_aggregate.h +++ b/program/src/oracle/upd_aggregate.h @@ -1,7 +1,7 @@ #pragma once #include "oracle.h" -#include "model/model.h" +#include "model/price_model.h" #include "model/price_model.c" /* FIXME: HACK TO DEAL WITH DOCKER LINKAGE ISSUES */ #include "pd.h" From 91087e934c25f064d1d72a6c45d4cf4b0581104a Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 09:37:08 -0600 Subject: [PATCH 15/21] Generic utilities for efficient code usable on and off chain with standalone off chain test suite --- program/src/oracle/util/avg.h | 168 +++++++++ program/src/oracle/util/clean | 3 + program/src/oracle/util/prng.h | 366 ++++++++++++++++++++ program/src/oracle/util/run_tests | 55 +++ program/src/oracle/util/test_align.c | 77 ++++ program/src/oracle/util/test_avg.c | 143 ++++++++ program/src/oracle/util/test_hash.c | 59 ++++ program/src/oracle/util/test_prng.c | 177 ++++++++++ program/src/oracle/util/test_prng_battery.c | 92 +++++ program/src/oracle/util/test_round.c | 31 ++ program/src/oracle/util/test_sar.c | 49 +++ program/src/oracle/util/util.h | 11 + 12 files changed, 1231 insertions(+) create mode 100644 program/src/oracle/util/avg.h create mode 100755 program/src/oracle/util/clean create mode 100644 program/src/oracle/util/prng.h create mode 100755 program/src/oracle/util/run_tests create mode 100644 program/src/oracle/util/test_align.c create mode 100644 program/src/oracle/util/test_avg.c create mode 100644 program/src/oracle/util/test_hash.c create mode 100644 program/src/oracle/util/test_prng.c create mode 100644 program/src/oracle/util/test_prng_battery.c create mode 100644 program/src/oracle/util/test_round.c create mode 100644 program/src/oracle/util/test_sar.c create mode 100644 program/src/oracle/util/util.h diff --git a/program/src/oracle/util/avg.h b/program/src/oracle/util/avg.h new file mode 100644 index 000000000..9fe8cfdc4 --- /dev/null +++ b/program/src/oracle/util/avg.h @@ -0,0 +1,168 @@ +#ifndef _pyth_oracle_util_avg_h_ +#define _pyth_oracle_util_avg_h_ + +/* Portable robust integer averaging. (No intermediate overflow, no + assumptions about platform type wides, no assumptions about integer + signed right shift, no signed integer division, no implict + conversions, etc.) */ + +#include "sar.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* uint8_t a = avg_2_uint8 ( x, y ); int8_t a = avg_2_int8 ( x, y ); + uint16_t a = avg_2_uint16( x, y ); int16_t a = avg_2_int16( x, y ); + uint32_t a = avg_2_uint32( x, y ); int32_t a = avg_2_int32( x, y ); + uint64_t a = avg_2_uint64( x, y ); int64_t a = avg_2_int64( x, y ); + + compute the average (a) of x and y of the corresponding type with <1 + ulp error (floor / round toward negative infinity rounding). That + is, these compute this exactly: + floor( (x+y)/2 ) + On systems where signed right shifts are sign extending, this is + identical to: + (x+y) >> 1 + but the below will be safe against intermediate overflow. */ + +static inline uint8_t avg_2_uint8 ( uint8_t x, uint8_t y ) { return (uint8_t )((((uint16_t)x)+((uint16_t)y))>>1); } +static inline uint16_t avg_2_uint16( uint16_t x, uint16_t y ) { return (uint16_t)((((uint32_t)x)+((uint32_t)y))>>1); } +static inline uint32_t avg_2_uint32( uint32_t x, uint32_t y ) { return (uint32_t)((((uint64_t)x)+((uint64_t)y))>>1); } + +static inline uint64_t +avg_2_uint64( uint64_t x, + uint64_t y ) { + + /* x+y might have intermediate overflow and we don't have an efficient + portable wider integer type available. So we can't just do + (x+y)>>1 to compute floor((x+y)/2). */ + +# if 1 /* Fewer ops but less parallel issue */ + /* Emulate an adc (add with carry) to get the 65-bit wide intermediate + and then shift back in as necessary */ + uint64_t z = x + y; + uint64_t c = (uint64_t)(z>1) + (c<<63); +# else /* More ops but more parallel issue */ + /* floor(x/2)+floor(y/2)==(x>>1)+(y>>1) doesn't have intermediate + overflow but it has two rounding errors. Noting that + (x+y)/2 == floor(x/2) + floor(y/2) + (x_lsb+y_lsb)/2 + == (x>>1) + (y>>1) + (x_lsb+y_lsb)/2 + implies we should increment when x and y have their lsbs set. */ + return (x>>1) + (y>>1) + (x & y & UINT64_C(1)); +# endif + +} + +static inline int8_t avg_2_int8 ( int8_t x, int8_t y ) { return (int8_t )sar_int16((int16_t)(((int16_t)x)+((int16_t)y)),1); } +static inline int16_t avg_2_int16 ( int16_t x, int16_t y ) { return (int16_t)sar_int32((int32_t)(((int32_t)x)+((int32_t)y)),1); } +static inline int32_t avg_2_int32 ( int32_t x, int32_t y ) { return (int32_t)sar_int64((int64_t)(((int64_t)x)+((int64_t)y)),1); } + +static inline int64_t +avg_2_int64( int64_t x, + int64_t y ) { + + /* Similar considerations as above */ + +# if 1 /* Fewer ops but less parallel issue */ + /* x+y = x+2^63 + y+2^63 - 2^64 ... exact ops + = ux + uy - 2^64 ... exact ops where ux and uy are exactly representable as a 64-bit uint + = uz + 2^64 c - 2^64 ... exact ops where uz is a 64-bit uint and c is in [0,1]. + Thus, as before + uz = ux+uy ... c ops + c = (uz=ux) ... exact ops + Noting that (ux + uy) mod 2^64 = (x+y+2^64) mod 2^64 = (x+y) mod 2^64 + and that signed and added adds are the same operation binary in twos + complement math yields: + uz = (uint64_t)(x+y) ... c ops + b = uz>=(x+2^63) ... exact ops + as we know x+2^63 is in [0,2^64), x+2^63 = x+/-2^63 mod 2^64. And + using the signed and unsigned adds are the same operation binary + in we have: + x+2^63 ... exact ops + ==x+/-2^63 mod 2^64 ... exact ops + ==x+/-2^63 ... c ops */ + uint64_t t = (uint64_t)x; + uint64_t z = t + (uint64_t)y; + uint64_t b = (uint64_t)(z>=(t+(((uint64_t)1)<<63))); + return (int64_t)((z>>1) - (b<<63)); +# else /* More ops but more parallel issue (significant more ops if no platform arithmetic right shift) */ + /* See above for uint64_t for details on this calc. */ + return sar_int64( x, 1 ) + sar_int64( y, 1 ) + (x & y & (int64_t)1); +# endif + +} + +/* uint8_t a = avg_uint8 ( x, n ); int8_t a = avg_int8 ( x, n ); + uint16_t a = avg_uint16( x, n ); int16_t a = avg_int16( x, n ); + uint32_t a = avg_uint32( x, n ); int32_t a = avg_int32( x, n ); + + compute the average (a) of the n element array f the corresponding + type pointed to by x with <1 ulp round error (truncate / round toward + zero rounding). Supports arrays with up to 2^32-1 elements. Returns + 0 for arrays with 0 elements. Elements of the array are unchanged by + this function. No 64-bit equivalents are provided here as that + requires a different and slower implementation (at least if + portability is required). */ + +#define PYTH_ORACLE_UTIL_AVG_DECL(func,T) /* Comments for uint32_t but apply for uint8_t and uint16_t too */ \ +static inline T /* == (1/n) sum_i x[i] with one rounding error and round toward zero rounding */ \ +func( T const * x, /* Indexed [0,n) */ \ + uint32_t n ) { /* Returns 0 on n==0 */ \ + if( !n ) return (T)0; /* Handle n==0 case */ \ + /* At this point n is in [1,2^32-1]. Sum up all the x into a 64-bit */ \ + /* wide signed accumulator. */ \ + uint64_t a = (uint64_t)0; \ + for( uint32_t r=n; r; r-- ) a += (uint64_t)*(x++); \ + /* At this point a is in [ 0, (2^32-1)(2^32-1) ] < 2^64 and thus was */ \ + /* computed without intermediate overflow. Complete the average. */ \ + return (T)( a / (uint64_t)n ); \ +} + +PYTH_ORACLE_UTIL_AVG_DECL( avg_uint8, uint8_t ) +PYTH_ORACLE_UTIL_AVG_DECL( avg_uint16, uint16_t ) +PYTH_ORACLE_UTIL_AVG_DECL( avg_uint32, uint32_t ) +/* See note above about 64-bit variants */ + +#undef PYTH_ORACLE_UTIL_AVG_DECL + +#define PYTH_ORACLE_UTIL_AVG_DECL(func,T) /* Comments for int32_t but apply for int8_t and int16_t too */ \ +static inline T /* == (1/n) sum_i x[i] with one rounding error and round toward zero rounding */ \ +func( T const * x, /* Indexed [0,n) */ \ + uint32_t n ) { /* Returns 0 on n==0 */ \ + if( !n ) return (T)0; /* Handle n==0 case */ \ + /* At this point n is in [1,2^32-1]. Sum up all the x into a 64-bit */ \ + /* wide signed accumulator. */ \ + int64_t a = (int64_t)0; \ + for( uint32_t r=n; r; r-- ) a += (int64_t)*(x++); \ + /* At this point a is in [ -2^31 (2^32-1), (2^31-1)(2^32-1) ] such that */ \ + /* |a| < 2^63. It thus was computed without intermediate overflow and */ \ + /* further |a| can fit with an uint64_t. Compute |a|/n. If a<0, the */ \ + /* result will be at most 2^31-1 (i.e. all x[i]==2^31-1). Otherwise, */ \ + /* the result will be at most 2^31 (i.e. all x[i]==-2^31). */ \ + int s = a<(int64_t)0; \ + uint64_t u = (uint64_t)(s ? -a : a); \ + u /= (uint64_t)n; \ + a = (int64_t)u; \ + return (T)(s ? -a : a); \ +} + +PYTH_ORACLE_UTIL_AVG_DECL( avg_int8, int8_t ) +PYTH_ORACLE_UTIL_AVG_DECL( avg_int16, int16_t ) +PYTH_ORACLE_UTIL_AVG_DECL( avg_int32, int32_t ) +/* See note above about 64-bit variants */ + +#undef PYTH_ORACLE_UTIL_AVG_DECL + +#ifdef __cplusplus +} +#endif + +#endif /* _pyth_oracle_util_avg_h_ */ + diff --git a/program/src/oracle/util/clean b/program/src/oracle/util/clean new file mode 100755 index 000000000..860ef1641 --- /dev/null +++ b/program/src/oracle/util/clean @@ -0,0 +1,3 @@ +#!/bin/sh +rm -rfv bin + diff --git a/program/src/oracle/util/prng.h b/program/src/oracle/util/prng.h new file mode 100644 index 000000000..57d71138c --- /dev/null +++ b/program/src/oracle/util/prng.h @@ -0,0 +1,366 @@ +#ifndef _pyth_oracle_util_prng_h_ +#define _pyth_oracle_util_prng_h_ + +/* Simple fast high quality non-cryptographic pseudo random number + generator. Supports parallel generation, interprocess shared memory + usage, checkpointing, random access, reversible, atomic, etc. Passes + extremely strict tests of randomness. + + Assumes hash.h provides a high quality 64<>64-bit integer hash + functions (i.e. full avalanche) with the property hash_uint64(0)==0 + and hash_uint64(i) for i in [0,2^64) yields a permutation of [0,2^64) + and also a reasonable efficient inverse. */ + +#include +#include "hash.h" /* includes stdint.h */ + +/* prng_t should be treated as an opaque handle of a pseudo random + number generator. (It technically isn't here to faciliate inlining + of prng operations.) */ + +struct prng { + uint64_t seq; + uint64_t idx; +}; + +typedef struct prng prng_t; + +#ifdef __cplusplus +extern "C" { +#endif + +/* Private: randomly expands an arbitrary 32-bit value into a unique + 64-bit non-sparse value such that the original 32-bit value can be + recovered and that 0 expands to something non-zero (the non-sparse + expansion helps reduce correlations between different sequences, the + zero to non-zero expansion means the common initialization of seq=0, + idx=0 doesn't yield 0 for the first random value as would happen for + hash functions that have the property hash_uint64(0)==0, the XOR + 64-bit const here is zero in the lower 32-bits and an arbitrary + non-zero in the upper 32-bits). For the current hash_uint64 + implementation and XOR 64-bit constant, the pop count of the expanded + seq is ~32.000 +/- ~4.000 and in [8,56] for all possible values of + seq (i.e. the expanded seq popcount is well approximated as a normal + with mean 32 and rms 4 and the extremes are in line with the expected + extremes for 2^32 samples). */ + +static inline uint64_t +prng_private_expand( uint32_t seq ) { + return hash_uint64( UINT64_C( 0x900df00d00000000 ) ^ (uint64_t)seq ); +} + +/* Private: extract the original 32-bit seq from its expanded value */ + +static inline uint32_t +prng_private_contract( uint64_t seq ) { + return (uint32_t)hash_inverse_uint64( seq ); +} + +/* prng_footprint and prng_align give the needed footprint and alignment + for a memory region to hold a prng's state. malloc / alloca / + declaration friendly (e.g. a memory region declared as "prng_t + _prng[1];", or created by "malloc(sizeof(prng_t))" or created by + "alloca(sizeof(prng_t))" will all automatically have the needed + footprint and alignment). + + prng_new takes ownership of the memory region pointed to by mem + (which is assumed to be non-NULL with the appropriate footprint and + alingment) and formats it as a prng. The pseudo random number + generator stream will initialized to use sequence random sequence + "seq" and will start at index "idx". Returns mem (which will be + formatted for use). The caller will not be joined to the region on + return. + + prng_join joins the caller to a memory region holding the state of a + prng. _prng points to a memory region in the local address space + that holds prng. Returns an opaque handle of the local join in the + local address space to the prng (which might not be the same thing as + _prng ... the separation of new and join is to facilitate + interprocess shared memory usage patterns while supporting + transparent upgrades users of this to more elaborate PRNG algorithms + where address translations under the hood may not be trivial). + + prng_leave leaves the current prng join. Returns a pointer in the + local address space to the memory region holding the state of the + prng. The prng join should not be used afterward. + + prng_delete unformats the memory region currently used to hold the + state of a _prng and returns ownership of the underlying memory + region to the caller. There should be no joins in the system on the + prng. Returns a pointer to the underlying memory region. + + FIXME: CONSIDER FLATTENING ALIGN? */ + +static inline uint64_t prng_footprint( void ) { return (uint64_t)sizeof ( prng_t ); } +static inline uint64_t prng_align ( void ) { return (uint64_t)alignof( prng_t ); } + +static inline void * +prng_new( void * mem, + uint32_t seq, + uint64_t idx ) { + prng_t * prng = (prng_t *)mem; + prng->seq = prng_private_expand( seq ); + prng->idx = idx; + return (void *)prng; +} + +static inline prng_t * prng_join ( void * _prng ) { return (prng_t *)_prng; } +static inline void * prng_leave ( prng_t * prng ) { return (void *) prng; } +static inline void * prng_delete( void * _prng ) { return (void *)_prng; } + +/* prng_seq returns the sequence used by the prng. prng_idx returns the + next index that will be consumed by the prng. */ + +static inline uint32_t prng_seq( prng_t * prng ) { return prng_private_contract( prng->seq ); } +static inline uint64_t prng_idx( prng_t * prng ) { return prng->idx; } + +/* prng_seq_set sets the sequence to be used by the prng and returns the + replaced value. prng_idx_set sets the next index that will be + consumed by the prng and returns the replaced value. */ + +static inline uint32_t +prng_seq_set( prng_t * prng, + uint32_t seq ) { + uint32_t old = prng_seq( prng ); + prng->seq = prng_private_expand( seq ); + return old; +} + +static inline uint64_t +prng_idx_set( prng_t * prng, + uint64_t idx ) { + uint64_t old = prng_idx( prng ); + prng->idx = idx; + return old; +} + +/* prng_uintN returns the next integer in the PRNG sequence in [0,2^N) + with uniform probability with a period of 2^64 (prng_uint64 has a + period of 2^63 as consumes two prng indices). Passes various strict + PRNG tests (e.g. diehard, dieharder, testu01, etc). Assumes prng is + a current join. prng_intN is the same as prng_uintN but returns a + value in [0,2^(N-1)). (A signed generator that can assume all + possible values of of a signed int uniform IID can be obtained by + casting the output of the unsigned generator of the same, assuming + typical twos complement arithmetic platform.) + + The theory for this that hash_uint64(i) for i in [0,2^64) specifies a + random looking permutation of the integers in [0,2^64). Returning + the low order bits of this random permutation then yields a high + quality non-cryptographic random number stream automatically as it: + + - Has a long period (2^64). This is implied by the permutation + property. + + - Has the expected random properties (as theoretically best possible + for a finite period generator) of a true uniform IID bit source. + For example, the probability of next random number is uniform and + independent of previous N random numbers for N<<2^64). This is + also implied by the full avalanche and permutation property. + + - Is "unpredictable". That is, the internal state of the generator + is difficult to recover from its outputs, e.g. a return from + prng_uint32 could be have been generated from 2^32 internal states + (if the sequence is known), up to 2^32 sequences (if the state is + known) and up to 2^64 (state,seq) pairs neither is known (the state + / sequence is potentially recoverable given a long enough stream of + values though). This is implied by the truncation of hash values. + + To turn this into parameterizable family of generators, note that + hash_uint64( perm_j( i ) ) where j is some parameterized family of + random permutations is still a permutation and would have all the + above properties for free so long as no perm_j is similar to the + inverse of the hash permutation. Practically, xoring i by a + non-sparse 64-bit number will ultra cheaply generate a wide family of + "good enough" permutations to do a parameterized shuffling of the + original hash_uint64 permutation, creating a large number of parallel + sequences. Since users are empirically notoriously bad at seeding + though, we only let the user specify a 32-bit sequence id and then + generate a unique non-sparse 64-bit random looking number from it. */ + +static inline uint8_t prng_uint8 ( prng_t * prng ) { return (uint8_t )hash_uint64( prng->seq ^ (prng->idx++) ); } +static inline uint16_t prng_uint16( prng_t * prng ) { return (uint16_t)hash_uint64( prng->seq ^ (prng->idx++) ); } +static inline uint32_t prng_uint32( prng_t * prng ) { return (uint32_t)hash_uint64( prng->seq ^ (prng->idx++) ); } + +static inline uint64_t +prng_uint64( prng_t * prng ) { + uint64_t hi = (uint64_t)prng_uint32( prng ); + return (hi<<32) | (uint64_t)prng_uint32( prng ); +} + +static inline int8_t prng_int8 ( prng_t * prng ) { return (int8_t )( prng_uint8 ( prng ) >> 1 ); } +static inline int16_t prng_int16( prng_t * prng ) { return (int16_t)( prng_uint16( prng ) >> 1 ); } +static inline int32_t prng_int32( prng_t * prng ) { return (int32_t)( prng_uint32( prng ) >> 1 ); } +static inline int64_t prng_int64( prng_t * prng ) { return (int64_t)( prng_uint64( prng ) >> 1 ); } + +/* These quickly and robustly convert uniform random integers into + uniform random floating point with appropriate rounding. Intervals + are: + + *_o -> (0,1) + *_c0 -> [0,1) + *_c1 -> (0,1] + *_c -> [0,1] + + To provide more specifics, let the real numbers from [0,1) be + partitioned into N uniform disjoint intervals such that interval i + goes from [i/N,(i+1)/N) where i is in [0,N). For single (double) + precision, "float" ("double"), the largest N for which the range of + each interval is _exactly_ representable is N = 2^24 (2^53). + + Given then a uniform IID uint32_t random input, the + prng_uint32_to_float_c0 converter output is as though an continuous + IID uniform random in [0,1) was generated and then rounded down to + the start of the containing interval (2^24 intervals). As such, this + generator can never return exactly 1 but it can exactly return +0. + Since floats have higher resolution near 0 than 1, this will not + return all float possible representations in [0,1) but it can return + all possible float representations in [1/2,1). In particular, this + will never return a denorm or -0. + + Similarly for prng_uint32_to_float_c1 converter rounds up to the end + of the containing interval / start of the next interval (2^24 + intervals). As such, this converter can never return exactly +0 but + it can exactly return 1. It will not return all possible float + representations in (0,1] but it can return all possible float + representations in [1/2,1]. This will never return a denorm or -0. + + The prng_uint32_to_float_c converter rounds toward nearest even + toward the start containing interval or start of the next interval + (2^24 intervals). As such, this can return both exactly +0 and + exactly 1 (and the probability of returning exactly +0 == probability + of exactly 1 == (1/2) probability all other possible return values). + It will not return all possible float representations in [0,1] but it + can return all float possible representations in [1/2,1]. This will + never return a denorm or -0. + + The prng_uint32_to_float_o converter rounds toward the middle of + containing internal (2^23 intervals ... note that then in a sense + this converter is 1-bit less accurate than the above). As such, this + can neither return +0 nor 1 and will not return all possible float + representations in (0,1). This will never return a denorm or -0. + + Similarly for the double variants (*_{c0,c1,c} uses 2^53 intervals + and o uses 2^52 intervals). + + Note that it is possible to make converters that will handle exactly + rounding toward all possible floating point representations in [0,1] + but this are significantly more expensive. + + Assumes IEEE-754 style float and doubles. + + FIXME: ADD UNIT TEST COVERAGE */ + +static inline float +prng_uint32_to_float_c0( uint32_t u ) { + return (1.f/(float )(INT32_C(1)<<24))*(float)(int32_t)( u>>(32-24) ); +} + +static inline float +prng_uint32_to_float_c1( uint32_t u ) { + return (1.f/(float )(INT32_C(1)<<24))*(float)(int32_t)((u>>(32-24))+ UINT32_C(1) ); +} + +static inline float +prng_uint32_to_float_c( uint32_t u ) { + return (1.f/(float )(INT32_C(1)<<24))*(float)(int32_t)((u>>(32-24))+(u&UINT32_C(1))); +} + +static inline float +prng_uint32_to_float_o( uint32_t u ) { + return (1.f/(float )(INT32_C(1)<<24))*(float)(int32_t)((u>>(32-24))| UINT32_C(1) ); +} + +static inline double +prng_uint64_to_double_c0( uint64_t u ) { + return (1. /(double)(INT64_C(1)<<53))*(double)(int64_t)( u>>(64-53) ); +} + +static inline double +prng_uint64_to_double_c1( uint64_t u ) { + return (1. /(double)(INT64_C(1)<<53))*(double)(int64_t)((u>>(64-53))+ UINT64_C(1) ); +} + +static inline double +prng_uint64_to_double_c( uint64_t u ) { + return (1. /(double)(INT64_C(1)<<53))*(double)(int64_t)((u>>(64-53))+(u&UINT64_C(1))); +} + +static inline double +prng_uint64_to_double_o( uint64_t u ) { + return (1. /(double)(INT64_C(1)<<53))*(double)(int64_t)((u>>(64-53))| UINT64_C(1) ); +} + +static inline float prng_float_c0 ( prng_t * prng ) { return prng_uint32_to_float_c0 ( prng_uint32( prng ) ); } +static inline float prng_float_c1 ( prng_t * prng ) { return prng_uint32_to_float_c1 ( prng_uint32( prng ) ); } +static inline float prng_float_c ( prng_t * prng ) { return prng_uint32_to_float_c ( prng_uint32( prng ) ); } +static inline float prng_float_o ( prng_t * prng ) { return prng_uint32_to_float_o ( prng_uint32( prng ) ); } + +static inline double prng_double_c0( prng_t * prng ) { return prng_uint64_to_double_c0( prng_uint64( prng ) ); } +static inline double prng_double_c1( prng_t * prng ) { return prng_uint64_to_double_c1( prng_uint64( prng ) ); } +static inline double prng_double_c ( prng_t * prng ) { return prng_uint64_to_double_c ( prng_uint64( prng ) ); } +static inline double prng_double_o ( prng_t * prng ) { return prng_uint64_to_double_o ( prng_uint64( prng ) ); } + +/* prng_int32_roll uses the given prng to roll an n-sided die where n + is the number of sides (assumed to be positive). That is returns + uniform IID rand in [0,n) even if n is not an exact power of two. + + Rejection method based. Specifically, the number of prng slots + consumed is typically 1 but theoretically might be higher + occassionally (64-bit wide types consume prng slots twice as fast). + + Deterministic prng slot consumption possible with a slightly more + approximate implementation (bound the number of iterations such that + this always consumes a fixed number of slot and accept the + practically infinitesimal bias when n is not a power of 2). + + FIXME: ADD UNIT TEST COVERAGE. */ + +static inline uint32_t +prng_private_roll32( prng_t * prng, + uint32_t n ) { + uint32_t r = (-n) % n; /* Compute 2^32 mod n = (2^32 - n) mod n = (-n) mod n, compile time for compile time n */ + uint32_t u; do u = prng_uint32( prng ); while( u~20 min (seq 0, idx 0) +#bin/test_prng_battery 4 0 0 || exit 1 # BigCrush: Takes >~3 hours (seq 0, idx 0) + +echo all tests passed + diff --git a/program/src/oracle/util/test_align.c b/program/src/oracle/util/test_align.c new file mode 100644 index 000000000..eaa55767b --- /dev/null +++ b/program/src/oracle/util/test_align.c @@ -0,0 +1,77 @@ +#include +#include +#include "util.h" + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + uint32_t shift_mask = (uint32_t)(sizeof(uintptr_t)*(size_t)CHAR_BIT); + if( !align_ispow2( (uintptr_t)shift_mask ) ) { + printf( "FAIL (platform)\n" ); + return 1; + } + shift_mask--; + + prng_t _prng[1]; + prng_t * prng = prng_join( prng_new( _prng, (uint32_t)0, (uint64_t)0 ) ); + + int ctr = 0; + for( int i=0; i<1000000000; i++ ) { + if( !ctr ) { printf( "Completed %i iterations\n", i ); ctr = 10000000; } + ctr--; + + /* Test align_ispow2 */ + + do { + uint32_t r = prng_uint32( prng ); + uint32_t s = (r>>8) & shift_mask; /* s is uniform IID valid right shift amount for a uintptr_t */ + r &= (uint32_t)255; /* r is 8-bit uniform IID rand (power of 2 with prob ~8/256=1/32) */ + uintptr_t x = ((uintptr_t)r) << s; /* x is power of 2 with prob ~1/32 */ + + int c = align_ispow2( x ); + int d = (x>(uintptr_t)0) && !(x & (x-(uintptr_t)1)); + if( c!=d ) { + printf( "FAIL (x %lx c %i d %i)\n", (unsigned long)x, c, d ); + return 1; + } + } while(0); + + /* Test align_up/align_dn/align_isaligned */ + + do { + uintptr_t a = ((uintptr_t)1) << (prng_uint32( prng ) & shift_mask); + + uintptr_t x = (uintptr_t)prng_uint64( prng ); + void * u = (void *)x; + void * v = align_dn( u, a ); + void * w = align_up( u, a ); + uintptr_t y = (uintptr_t)v; + uintptr_t z = (uintptr_t)w; + + int already_aligned = (u==v); + int already_aligned_1 = (u==w); + + uintptr_t mask = a-(uintptr_t)1; + if( already_aligned != already_aligned_1 || + align_isaligned( u, a )!=already_aligned || + !align_isaligned( v, a ) || + !align_isaligned( w, a ) || + ( x & ~mask) != y || + ((x+mask) & ~mask) != z ) { + printf( "FAIL (a %lx aa %i aa1 %i x %lx y %lx z %lx)\n", + (unsigned long)a, already_aligned, already_aligned_1, (unsigned long)x, (unsigned long)y, (unsigned long)z ); + return 1; + } + + } while(0); + } + + prng_delete( prng_leave( prng ) ); + + printf( "pass\n" ); + + return 0; +} + diff --git a/program/src/oracle/util/test_avg.c b/program/src/oracle/util/test_avg.c new file mode 100644 index 000000000..21f042449 --- /dev/null +++ b/program/src/oracle/util/test_avg.c @@ -0,0 +1,143 @@ +#include +#include "util.h" + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + prng_t _prng[1]; + prng_t * prng = prng_join( prng_new( _prng, (uint32_t)0, (uint64_t)0 ) ); + + int ctr; + + ctr = 0; + for( int i=0; i<1000000000; i++ ) { + if( !ctr ) { printf( "reg: Completed %i iterations\n", i ); ctr = 10000000; } + ctr--; + +# define TEST(w) do { \ + uint##w##_t x = prng_uint##w( prng ); \ + uint##w##_t y = prng_uint##w( prng ); \ + uint##w##_t z = avg_2_uint##w( x, y ); \ + uint##w##_t r = (uint##w##_t)(((uint64_t)x+(uint64_t)y) >> 1); \ + if( z!=r ) { \ + printf( "FAIL (iter %i op avg_2_uint" #w "x %lu y %lu z %lu r %lu\n", \ + i, (long unsigned)x, (long unsigned)y, (long unsigned)z, (long unsigned)r ); \ + return 1; \ + } \ + } while(0) + TEST(8); + TEST(16); + TEST(32); +# undef TEST + + do { + uint64_t x = prng_uint64( prng ); + uint64_t y = prng_uint64( prng ); + uint64_t z = avg_2_uint64( x, y ); + uint64_t r = (x>>1) + (y>>1) + (x & y & (uint64_t)1); + if( z!=r ) { + printf( "FAIL (iter %i op avg_2_uint64 x %lu y %lu z %lu r %lu\n", + i, (long unsigned)x, (long unsigned)y, (long unsigned)z, (long unsigned)r ); + return 1; + } + } while(0); + \ +# define TEST(w) do { \ + int##w##_t x = (int##w##_t)prng_uint##w( prng ); \ + int##w##_t y = (int##w##_t)prng_uint##w( prng ); \ + int##w##_t z = avg_2_int##w( x, y ); \ + int##w##_t r = (int##w##_t)(((int64_t)x+(int64_t)y) >> 1); \ + if( z!=r ) { \ + printf( "FAIL (iter %i op avg_2_int" #w "x %li y %li z %li r %li\n", \ + i, (long)x, (long)y, (long)z, (long)r ); \ + return 1; \ + } \ + } while(0) + TEST(8); + TEST(16); + TEST(32); +# undef TEST + + do { + int64_t x = (int64_t)prng_uint64( prng ); + int64_t y = (int64_t)prng_uint64( prng ); + int64_t z = avg_2_int64( x, y ); + int64_t r = (x>>1) + (y>>1) + (x & y & (int64_t)1); + if( z!=r ) { + printf( "FAIL (iter %i op avg_2_int64 x %li y %li z %li r %li\n", + i, (long)x, (long)y, (long)z, (long)r ); + return 1; + } + } while(0); + } + +# define N 512 + + ctr = 0; + for( int i=0; i<10000000; i++ ) { + if( !ctr ) { printf( "mem: Completed %i iterations\n", i ); ctr = 100000; } + ctr--; + +# define TEST(w) do { \ + uint##w##_t x[N]; \ + uint32_t n = prng_uint32( prng ) & (uint32_t)(N-1); \ + uint64_t a = (uint64_t)0; \ + for( uint32_t i=(uint32_t)0; i +#include "util.h" + +uint32_t ref32[10] = { + UINT32_C( 0x00000000 ), + UINT32_C( 0x514e28b7 ), + UINT32_C( 0x30f4c306 ), + UINT32_C( 0x85f0b427 ), + UINT32_C( 0x249cb285 ), + UINT32_C( 0xcc0d53cd ), + UINT32_C( 0x5ceb4d08 ), + UINT32_C( 0x18c9aec4 ), + UINT32_C( 0x4939650b ), + UINT32_C( 0xc27c2913 ) +}; + + +uint64_t ref64[10] = { + UINT64_C( 0x0000000000000000 ), + UINT64_C( 0xb456bcfc34c2cb2c ), + UINT64_C( 0x3abf2a20650683e7 ), + UINT64_C( 0x0b5181c509f8d8ce ), + UINT64_C( 0x47900468a8f01875 ), + UINT64_C( 0xd66ad737d54c5575 ), + UINT64_C( 0xe8b4b3b1c77c4573 ), + UINT64_C( 0x740729cbe468d1dd ), + UINT64_C( 0x46abcca593a3c687 ), + UINT64_C( 0x91209a1ff7f4f1d5 ) +}; + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + for( int i=0; i<10; i++ ) { + uint32_t x = (uint32_t)i; + uint32_t y = hash_uint32( x ); + uint32_t z = hash_inverse_uint32( y ); + if( y!=ref32[i] ) { printf( "ref32: FAIL\n" ); return 1; } + if( x!=z ) { printf( "inv32: FAIL\n" ); return 1; } + if( hash_uint32( hash_inverse_uint32( x ) )!=x ) { printf( "INV32: FAIL\n" ); return 1; } + } + + for( int i=0; i<10; i++ ) { + uint64_t x = (uint64_t)i; + uint64_t y = hash_uint64( x ); + uint64_t z = hash_inverse_uint64( y ); + if( y!=ref64[i] ) { printf( "ref64: FAIL\n" ); return 1; } + if( x!=z ) { printf( "inv64: FAIL\n" ); return 1; } + if( hash_uint64( hash_inverse_uint64( x ) )!=x ) { printf( "INV64: FAIL\n" ); return 1; } + } + + /* FIXME: MEASURE AVALANCHE PROPERTIES? */ + + printf( "pass\n" ); + return 0; +} + diff --git a/program/src/oracle/util/test_prng.c b/program/src/oracle/util/test_prng.c new file mode 100644 index 000000000..20f65e360 --- /dev/null +++ b/program/src/oracle/util/test_prng.c @@ -0,0 +1,177 @@ +#include +#include +#include +#include "util.h" + +/* First 10 uint64 for seq 0 starting from idx 0 */ + +uint64_t x0_0[10] = { + UINT64_C( 0xa036f9b67313c1aa ), + UINT64_C( 0x110a4ea5e65927d2 ), + UINT64_C( 0x9ddf34cf83d17c94 ), + UINT64_C( 0xeb33d1b534e210ec ), + UINT64_C( 0x9b15b94d3e81b76a ), + UINT64_C( 0xee9544dab2bf64bf ), + UINT64_C( 0x5c4b0ccf7c94d274 ), + UINT64_C( 0xf0f83ab44262ad1f ), + UINT64_C( 0xf11b1aa14c7dabd6 ), + UINT64_C( 0x3800dde6d02d6ed7 ) +}; + +/* First 10 uint64 for seq 0 starting from idx 20 */ + +uint64_t x1_20[10] = { + UINT64_C( 0x55369a6a0817cbce ), + UINT64_C( 0xce1baeb695229132 ), + UINT64_C( 0x0e443e81e9e722d2 ), + UINT64_C( 0xc6c065484f76e825 ), + UINT64_C( 0x37dc474806fabc8a ), + UINT64_C( 0x9fa3305df1b56824 ), + UINT64_C( 0xf3b3961a17ed881c ), + UINT64_C( 0x646f40006cef8d6f ), + UINT64_C( 0x4d6955f607a153b2 ), + UINT64_C( 0xf806bccb58d0e60b ) +}; + +void +printf_ref( uint64_t * ref, + uint32_t seq, + uint64_t idx ) { + printf( "uint64_t x%u_%lu[10] = {\n", seq, idx ); + for( int i=0; i<10; i++ ) printf( " UINT64_C( 0x%016lx )%s\n", ref[i], i<9 ? "," : "" ); + printf( "};\n" ); +} + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + uint64_t x[10]; + + if( prng_footprint()!=(uint64_t)sizeof (prng_t) ) { printf( "FAIL (footprint)\n" ); return 1; } + if( prng_align ()!=(uint64_t)alignof(prng_t) ) { printf( "FAIL (align)\n" ); return 1; } + + prng_t _prng[1]; + prng_t * prng = prng_join( prng_new( _prng, (uint32_t)0, (uint64_t)0 ) ); + + if( prng_seq( prng )!=(uint32_t)0 ) { printf( "FAIL (seq)\n" ); return 1; } + if( prng_idx( prng )!=(uint64_t)0 ) { printf( "FAIL (idx)\n" ); return 1; } + + for( int i=0; i<10; i++ ) x[i] = prng_uint64( prng ); + for( int i=0; i<10; i++ ) + if( x[i]!=x0_0[i] ) { + printf( "FAIL(0,0)\n" ); + printf_ref( x0_0, (uint32_t)0, (uint64_t)0 ); + printf_ref( x, (uint32_t)0, (uint64_t)0 ); + return 1; + } + + if( prng_seq( prng )!=(uint32_t) 0 ) { printf( "FAIL (seq-1)\n" ); return 1; } + if( prng_idx( prng )!=(uint64_t)20 ) { printf( "FAIL (idx-1)\n" ); return 1; } + + if( prng_seq_set( prng, (uint32_t)1 )!=(uint32_t)0 ) { printf( "FAIL (seq_set)\n" ); return 1; } + + for( int i=0; i<10; i++ ) x[i] = prng_uint64( prng ); + for( int i=0; i<10; i++ ) + if( x[i]!=x1_20[i] ) { + printf( "FAIL(1,20)\n" ); + printf_ref( x1_20, (uint32_t)1, (uint64_t)20 ); + printf_ref( x, (uint32_t)1, (uint64_t)20 ); + return 1; + } + + if( prng_seq( prng )!=(uint32_t) 1 ) { printf( "FAIL (seq-2)\n" ); return 1; } + if( prng_idx( prng )!=(uint64_t)40 ) { printf( "FAIL (idx-2)\n" ); return 1; } + + if( prng_seq_set( prng, (uint32_t)0 )!=(uint32_t) 1 ) { printf( "FAIL (seq_set-1)\n" ); return 1; } + if( prng_idx_set( prng, (uint64_t)0 )!=(uint64_t)40 ) { printf( "FAIL (idx_set-1)\n" ); return 1; } + + for( int i=0; i<10; i++ ) x[i] = prng_uint64( prng ); + for( int i=0; i<10; i++ ) if( x[i]!=x0_0[i] ) { printf( "FAIL(0,0)-1\n" ); return 1; } + + uint64_t domain = (uint64_t)0; + for( int i=0; i<1000; i++ ) { + uint8_t u8 = prng_uint8 ( prng ); + uint16_t u16 = prng_uint16 ( prng ); + uint32_t u32 = prng_uint32 ( prng ); + uint64_t u64 = prng_uint64 ( prng ); + + int8_t i8 = prng_int8 ( prng ); if( i8 <(int8_t )0 ) { printf( "FAIL (int8 )\n" ); return 1; } + int16_t i16 = prng_int16 ( prng ); if( i16<(int16_t)0 ) { printf( "FAIL (int16)\n" ); return 1; } + int32_t i32 = prng_int32 ( prng ); if( i32<(int32_t)0 ) { printf( "FAIL (int32)\n" ); return 1; } + int64_t i64 = prng_int64 ( prng ); if( i64<(int64_t)0 ) { printf( "FAIL (int64)\n" ); return 1; } + + float fc0 = prng_float_c0 ( prng ); if( !( 0.f<=fc0 && fc0< 1.f) ) { printf( "FAIL (float_c0)\n" ); return 1; } + float fc1 = prng_float_c1 ( prng ); if( !( 0.f< fc1 && fc1<=1.f) ) { printf( "FAIL (float_c1)\n" ); return 1; } + float fc = prng_float_c ( prng ); if( !( 0.f<=fc && fc <=1.f) ) { printf( "FAIL (float_c)\n" ); return 1; } + float fnc = prng_float_o ( prng ); if( !( 0.f< fnc && fnc< 1.f) ) { printf( "FAIL (float_o)\n" ); return 1; } + + double dc0 = prng_double_c0( prng ); if( !( 0.f<=dc0 && dc0< 1.f) ) { printf( "FAIL (double_c0)\n" ); return 1; } + double dc1 = prng_double_c1( prng ); if( !( 0.f< dc1 && dc1<=1.f) ) { printf( "FAIL (double_c1)\n" ); return 1; } + double dc = prng_double_c ( prng ); if( !( 0.f<=dc && dc <=1.f) ) { printf( "FAIL (double_c)\n" ); return 1; } + double dnc = prng_double_o ( prng ); if( !( 0.f< dnc && dnc< 1.f) ) { printf( "FAIL (double_o)\n" ); return 1; } + +# define LO(i,w) ((uint64_t)((uint64_t)(i >> (w-4))==(uint64_t)0x0)) +# define HI(i,w) ((uint64_t)((uint64_t)(i >> (w-4))==(uint64_t)0xf)) + + domain |= LO(u8, 8 ) << 0; domain |= HI(u8, 8 ) << 4; + domain |= LO(u16,16) << 1; domain |= HI(u16,16) << 5; + domain |= LO(u32,32) << 2; domain |= HI(u32,32) << 6; + domain |= LO(u64,64) << 3; domain |= HI(u64,64) << 7; + + domain |= LO(i8, 7 ) << 8; domain |= HI(i8, 7 ) << 12; + domain |= LO(i16,15) << 9; domain |= HI(i16,15) << 13; + domain |= LO(i32,31) << 10; domain |= HI(i32,31) << 14; + domain |= LO(i64,63) << 11; domain |= HI(i64,63) << 15; + +# undef HI +# undef LO +# define LO(f) ((uint64_t)(f<0.0625f)) +# define HI(f) ((uint64_t)(f>0.9375f)) + + domain |= LO(fc0) << 16; domain |= HI(fc0) << 20; + domain |= LO(fc1) << 17; domain |= HI(fc1) << 21; + domain |= LO(fc ) << 18; domain |= HI(fc ) << 22; + domain |= LO(fnc) << 19; domain |= HI(fnc) << 23; + +# undef HI +# undef LO +# define LO(d) ((uint64_t)(d<0.0625)) +# define HI(d) ((uint64_t)(d>0.9375)) + + domain |= LO(dc0) << 24; domain |= HI(dc0) << 28; + domain |= LO(dc1) << 25; domain |= HI(dc1) << 29; + domain |= LO(dc ) << 26; domain |= HI(dc ) << 30; + domain |= LO(dnc) << 27; domain |= HI(dnc) << 31; + +# undef HI +# undef LO + } + if( domain!=((UINT64_C(1)<<32)-UINT64_C(1)) ) { printf( "FAIL (domain)\n" ); return 1; } + + prng_delete( prng_leave( prng ) ); + +# if 0 + long sum_pop = 0L; + long sum_pop2 = 0L; + int min_pop = INT_MAX; + int max_pop = INT_MIN; + for( long i=0; i<(1L<<32); i++ ) { + uint64_t seq = prng_private_expand( (uint32_t)i ); + if( prng_private_contract( seq )!=(uint32_t)i ) { printf( "FAIL (contract)\n" ); return 1; } + int pop = __builtin_popcountl( seq ); + sum_pop += (long) pop; + sum_pop2 += (long)(pop*pop); + min_pop = popmax_pop ? pop : max_pop; + } + double avg_pop = ((double)sum_pop ) / ((double)(1L<<32)); + double rms_pop = sqrt( (((double)sum_pop2) / ((double)(1L<<32))) - avg_pop*avg_pop ); + printf( "expand popcount stats: %.3f +/- %.3f [%i,%i]\n", avg_pop, rms_pop, min_pop, max_pop ); +# endif + + printf( "pass\n" ); + return 0; +} + diff --git a/program/src/oracle/util/test_prng_battery.c b/program/src/oracle/util/test_prng_battery.c new file mode 100644 index 000000000..41ffb9ee2 --- /dev/null +++ b/program/src/oracle/util/test_prng_battery.c @@ -0,0 +1,92 @@ +#include +#include +#include /* Assumes TestU01 install include directory is in the include search path */ +#include "util.h" + +static double +test_GetU01( void * param, + void * state ) { + (void)param; + return 2.3283064365386962890625e-10 /* 2^-32, exact */ * (double)prng_uint32( (prng_t *)state ); +} + +static unsigned long +test_GetBits( void * param, + void * state ) { + (void)param; + return (unsigned long)prng_uint32( (prng_t *)state ); +} + +static void +test_Write( void * state ) { + prng_t * prng = (prng_t *)state; + printf( "prng(0x%08lxU,0x%016lxUL)\n", (unsigned long)prng_seq( prng ), (unsigned long)prng_idx( prng ) ); +} + +static void +usage( char const * cmd ) { + fprintf( stderr, + "Usage: %s [bat] [seq] [idx]\n" + "\tbat 0 - FIPS-140.2 (fast)\n" + "\tbat 1 - Pseudo-DIEHARD (fast)\n" + "\tbat 2 - TestU01 SmallCrush (fast)\n" + "\tbat 3 - TestU01 Crush (~20 minutes)\n" + "\tbat 4 - TestU01 BigCrush (several hours)\n" + "Note that when running a test in a battery, the probability of it failing even\n" + "though the generator is fine is roughly 0.2%%. Thus, when repeatedly running\n" + "batteries that themselves contain a large number of tests, some test failures\n" + "are expected. So long as the test failures are are sporadic, don't occur in the\n" + "same place when running multiple times with random seq and/or idx, and don't\n" + "have p-values improbably close to 0 (p <<< 1/overall_num_tests_run) or 1\n" + "(1-p <<< 1/overall_num_tests_run), it is expected and normal\n", + cmd ); +} + +int +main( int argc, + char ** argv ) { + + /* Get command line arguments */ + + if( argc!=4 ) { usage( argv[0] ); return 1; } + int bat = atoi( argv[1] ); + uint32_t seq = (uint32_t)strtoul( argv[2], NULL, 0 ); + uint64_t idx = (uint64_t)strtoul( argv[3], NULL, 0 ); + + /* Create the test generator */ + + prng_t _prng[1]; + prng_t * prng = prng_join( prng_new( _prng, seq, idx ) ); + + /* Run the requested test battery */ + + char name[128]; + sprintf( name, "prng(0x%08lxU,0x%016lxUL)", (unsigned long)prng_seq( prng ), (unsigned long)prng_idx( prng ) ); + + unif01_Gen gen[1]; + gen->state = prng; + gen->param = NULL; + gen->name = name; + gen->GetU01 = test_GetU01; + gen->GetBits = test_GetBits; + gen->Write = test_Write; + + switch( bat ) { + case 0: bbattery_FIPS_140_2( gen ); break; + case 1: bbattery_pseudoDIEHARD( gen ); break; + case 2: bbattery_SmallCrush( gen ); break; + case 3: bbattery_Crush( gen ); break; + case 4: bbattery_BigCrush( gen ); break; +//case 5: bbattery_Rabbit( gen ); break; /* FIXME: NB */ +//case 6: bbattery_Alphabit( gen ); break; /* FIXME: NB/R/S */ +//case 7: bbattery_BlockAlphabit( gen ); break; /* FIXME: NB/R/S */ + default: usage( argv[0] ); return 1; + } + + /* Destroy the test generator */ + + prng_delete( prng_leave( prng ) ); + + return 0; +} + diff --git a/program/src/oracle/util/test_round.c b/program/src/oracle/util/test_round.c new file mode 100644 index 000000000..aff69822f --- /dev/null +++ b/program/src/oracle/util/test_round.c @@ -0,0 +1,31 @@ +#include + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + unsigned i = (unsigned)0; + + int ctr = 0; + for( int x=-32767; x<=32767; x++ ) { + for( int y=-32767; y<=32767; y++ ) { + if( !ctr ) { printf( "Completed %u iterations\n", i ); ctr = 10000000; } + ctr--; + + int u = (x+y)>>1; + int v = (x>>1)+(y>>1); + int w = v + (x&y&1); + if( u!=w ) { + printf( "FAIL (x %3i y %3i u %3i v %3i w %3i)\n", x, y, u, v, w ); + return 1; + } + + i++; + } + } + printf( "pass\n" ); + + return 0; +} + diff --git a/program/src/oracle/util/test_sar.c b/program/src/oracle/util/test_sar.c new file mode 100644 index 000000000..f0b1ac5ef --- /dev/null +++ b/program/src/oracle/util/test_sar.c @@ -0,0 +1,49 @@ +#include +#include "util.h" + +int +main( int argc, + char ** argv ) { + (void)argc; (void)argv; + + prng_t _prng[1]; + prng_t * prng = prng_join( prng_new( _prng, (uint32_t)0, (uint64_t)0 ) ); + + /* FIXME: EXPLICT COVERAGE OF EDGE CASES (PROBABLY STATICALLY FULLY + SAMPLED ALREADY THOUGH FOR 8 AND 16 BIT TYPES) */ + + int ctr = 0; + for( int i=0; i<1000000000; i++ ) { + if( !ctr ) { printf( "Completed %i iterations\n", i ); ctr = 10000000; } + ctr--; + + /* These tests assume the unit test platform has arithmetic right + shift for signed integers (and the diagnostic printfs assume that + long is at least a w-bit wide type) */ + +# define TEST_SAR(w) do { \ + int##w##_t x = (int##w##_t)prng_uint##w( prng ); \ + int n = (int)( prng_uint32( prng ) & (uint32_t)(w-1) ); \ + int##w##_t val = sar_int##w( x, n ); \ + int##w##_t ref = (int##w##_t)(x >> n); \ + if( val!=ref ) { \ + printf( "FAIL (iter %i op %i x %li n %i val %li ref %li)", i, w, (long)x, n, (long)val, (long)ref ); \ + return 1; \ + } \ + } while(0) + + TEST_SAR(8); + TEST_SAR(16); + TEST_SAR(32); + TEST_SAR(64); +# undef TEST_SAR + + } + + prng_delete( prng_leave( prng ) ); + + printf( "pass\n" ); + + return 0; +} + diff --git a/program/src/oracle/util/util.h b/program/src/oracle/util/util.h new file mode 100644 index 000000000..c6a179cc6 --- /dev/null +++ b/program/src/oracle/util/util.h @@ -0,0 +1,11 @@ +#ifndef _pyth_oracle_util_util_h_ +#define _pyth_oracle_util_util_h_ + +#include "align.h" /* includes stdint.h */ +//#include "sar.h" /* includes stdint.h */ +//#include "hash.h" /* includes stdint.h */ +#include "avg.h" /* includes sar.h */ +#include "prng.h" /* includes stdalign.h and hash.h */ + +#endif /* _pyth_oracle_util_util_h_ */ + From 71d08aa88711424d26ef0005c3b7240d5cba7164 Mon Sep 17 00:00:00 2001 From: Reisen Date: Thu, 17 Feb 2022 16:55:00 +0000 Subject: [PATCH 16/21] oracle: update test_oracle with new expected values --- program/src/oracle/test_oracle.c | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/program/src/oracle/test_oracle.c b/program/src/oracle/test_oracle.c index bdda372ba..0b74197ab 100644 --- a/program/src/oracle/test_oracle.c +++ b/program/src/oracle/test_oracle.c @@ -654,9 +654,9 @@ Test( oracle, upd_aggregate ) { px->comp_[px->num_++].latest_ = p2; px->comp_[px->num_++].latest_ = p1; upd_aggregate( px, 1001, 2 ); - cr_assert( px->agg_.price_ == 147 ); - cr_assert( px->agg_.conf_ == 48 ); - cr_assert( px->twap_.val_ == 108 ); + cr_assert( px->agg_.price_ == 145 ); + cr_assert( px->agg_.conf_ == 55 ); + cr_assert( px->twap_.val_ == 106 ); cr_assert( px->twac_.val_ == 16 ); cr_assert( px->num_qt_ == 2 ); cr_assert( px->timestamp_ == 2 ); @@ -673,15 +673,15 @@ Test( oracle, upd_aggregate ) { px->comp_[px->num_++].latest_ = p1; px->comp_[px->num_++].latest_ = p3; upd_aggregate( px, 1001, 3 ); - cr_assert( px->agg_.price_ == 191 ); - cr_assert( px->agg_.conf_ == 74 ); - cr_assert( px->twap_.val_ == 116 ); - cr_assert( px->twac_.val_ == 22 ); + cr_assert( px->agg_.price_ == 200 ); + cr_assert( px->agg_.conf_ == 90 ); + cr_assert( px->twap_.val_ == 114 ); + cr_assert( px->twac_.val_ == 23 ); cr_assert( px->num_qt_ == 3 ); cr_assert( px->timestamp_ == 3 ); cr_assert( px->prev_slot_ == 1000 ); - cr_assert( px->prev_price_ == 147 ); - cr_assert( px->prev_conf_ == 48 ); + cr_assert( px->prev_price_ == 145 ); + cr_assert( px->prev_conf_ == 55 ); cr_assert( px->prev_timestamp_ == 2 ); // four publishers @@ -693,16 +693,16 @@ Test( oracle, upd_aggregate ) { px->comp_[px->num_++].latest_ = p4; px->comp_[px->num_++].latest_ = p2; upd_aggregate( px, 1001, 4 ); - cr_assert( px->agg_.price_ == 235 ); - cr_assert( px->agg_.conf_ == 99 ); - cr_assert( px->twap_.val_ == 124 ); - cr_assert( px->twac_.val_ == 27 ); + cr_assert( px->agg_.price_ == 245 ); + cr_assert( px->agg_.conf_ == 85 ); + cr_assert( px->twap_.val_ == 125 ); + cr_assert( px->twac_.val_ == 28 ); cr_assert( px->last_slot_ == 1001 ); cr_assert( px->num_qt_ == 4 ); cr_assert( px->timestamp_ == 4 ); cr_assert( px->prev_slot_ == 1000 ); - cr_assert( px->prev_price_ == 191 ); - cr_assert( px->prev_conf_ == 74 ); + cr_assert( px->prev_price_ == 200 ); + cr_assert( px->prev_conf_ == 90 ); cr_assert( px->prev_timestamp_ == 3 ); upd_aggregate( px, 1025, 5 ); @@ -711,8 +711,8 @@ Test( oracle, upd_aggregate ) { cr_assert( px->num_qt_ == 4 ); cr_assert( px->timestamp_ == 5 ); cr_assert( px->prev_slot_ == 1001 ); - cr_assert( px->prev_price_ == 235 ); - cr_assert( px->prev_conf_ == 99 ); + cr_assert( px->prev_price_ == 245 ); + cr_assert( px->prev_conf_ == 85 ); cr_assert( px->prev_timestamp_ == 4 ); // check what happens when nothing publishes for a while From c63772acae019484549301a60d4890184cca1dda Mon Sep 17 00:00:00 2001 From: Kevin J Bowers Date: Tue, 11 Jan 2022 11:19:10 -0600 Subject: [PATCH 17/21] Corrected path --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7756bc60e..df5d49129 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ set( PC_SRC pc/request.cpp; pc/rpc_client.cpp; pc/user.cpp; - program/src/model/price_model.c + program/src/oracle/model/price_model.c ) set( PC_HDR From cc322b4ec1121a0556cc957538e68ad5230b93ff Mon Sep 17 00:00:00 2001 From: Reisen Date: Wed, 23 Feb 2022 12:17:52 +0000 Subject: [PATCH 18/21] tests: update qset results with new test values --- pyth/tests/qset/10.result | 2 +- pyth/tests/qset/11.result | 2 +- pyth/tests/qset/13.result | 2 +- pyth/tests/qset/14.result | 2 +- pyth/tests/qset/15.result | 2 +- pyth/tests/qset/16.result | 2 +- pyth/tests/qset/17.result | 2 +- pyth/tests/qset/18.result | 2 +- pyth/tests/qset/19.result | 2 +- pyth/tests/qset/20.result | 2 +- pyth/tests/qset/21.result | 2 +- pyth/tests/qset/22.result | 2 +- pyth/tests/qset/24.result | 2 +- pyth/tests/qset/25.result | 2 +- pyth/tests/qset/26.result | 2 +- pyth/tests/qset/27.result | 2 +- pyth/tests/qset/28.result | 2 +- pyth/tests/qset/29.result | 2 +- pyth/tests/qset/30.result | 2 +- pyth/tests/qset/31.result | 2 +- pyth/tests/qset/32.result | 2 +- pyth/tests/qset/33.result | 2 +- pyth/tests/qset/34.result | 2 +- pyth/tests/qset/35.result | 2 +- pyth/tests/qset/37.result | 2 +- pyth/tests/qset/4.result | 2 +- pyth/tests/qset/5.result | 2 +- pyth/tests/qset/6.result | 2 +- pyth/tests/qset/7.result | 2 +- pyth/tests/qset/8.result | 2 +- 30 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pyth/tests/qset/10.result b/pyth/tests/qset/10.result index 18e8ce21d..865784cfe 100644 --- a/pyth/tests/qset/10.result +++ b/pyth/tests/qset/10.result @@ -1 +1 @@ -{"exponent":-3,"price":500497,"conf":498,"status":"trading"} +{"exponent":-3,"price":501000,"conf":499989499000,"status":"trading"} diff --git a/pyth/tests/qset/11.result b/pyth/tests/qset/11.result index 3251f4c4e..f733efbdd 100644 --- a/pyth/tests/qset/11.result +++ b/pyth/tests/qset/11.result @@ -1 +1 @@ -{"exponent":-3,"price":500502460000,"conf":498770000,"status":"trading"} +{"exponent":-3,"price":500000000000,"conf":499999499990,"status":"trading"} diff --git a/pyth/tests/qset/13.result b/pyth/tests/qset/13.result index e48e704a4..7559a13a7 100644 --- a/pyth/tests/qset/13.result +++ b/pyth/tests/qset/13.result @@ -1 +1 @@ -{"exponent":-3,"price":13003,"conf":1845,"status":"trading"} +{"exponent":-3,"price":13005,"conf":1985,"status":"trading"} diff --git a/pyth/tests/qset/14.result b/pyth/tests/qset/14.result index af9d6da41..1b7033f0f 100644 --- a/pyth/tests/qset/14.result +++ b/pyth/tests/qset/14.result @@ -1 +1 @@ -{"exponent":-3,"price":10521,"conf":500,"status":"trading"} +{"exponent":-3,"price":11000,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/15.result b/pyth/tests/qset/15.result index c86ab32b8..748233c00 100644 --- a/pyth/tests/qset/15.result +++ b/pyth/tests/qset/15.result @@ -1 +1 @@ -{"exponent":-3,"price":13000,"conf":1750,"status":"trading"} +{"exponent":-3,"price":13000,"conf":2000,"status":"trading"} diff --git a/pyth/tests/qset/16.result b/pyth/tests/qset/16.result index d08b919e6..a77520155 100644 --- a/pyth/tests/qset/16.result +++ b/pyth/tests/qset/16.result @@ -1 +1 @@ -{"exponent":-3,"price":13128,"conf":1705,"status":"trading"} +{"exponent":-3,"price":13153,"conf":2053,"status":"trading"} diff --git a/pyth/tests/qset/17.result b/pyth/tests/qset/17.result index d08b919e6..a77520155 100644 --- a/pyth/tests/qset/17.result +++ b/pyth/tests/qset/17.result @@ -1 +1 @@ -{"exponent":-3,"price":13128,"conf":1705,"status":"trading"} +{"exponent":-3,"price":13153,"conf":2053,"status":"trading"} diff --git a/pyth/tests/qset/18.result b/pyth/tests/qset/18.result index 8e24d7fa3..d9ca2f29c 100644 --- a/pyth/tests/qset/18.result +++ b/pyth/tests/qset/18.result @@ -1 +1 @@ -{"exponent":-3,"price":11031,"conf":251,"status":"trading"} +{"exponent":-3,"price":11000,"conf":250,"status":"trading"} diff --git a/pyth/tests/qset/19.result b/pyth/tests/qset/19.result index 93c303464..5fcaed8b3 100644 --- a/pyth/tests/qset/19.result +++ b/pyth/tests/qset/19.result @@ -1 +1 @@ -{"exponent":-3,"price":11031,"conf":259,"status":"trading"} +{"exponent":-3,"price":10900,"conf":900,"status":"trading"} diff --git a/pyth/tests/qset/20.result b/pyth/tests/qset/20.result index ec58ce069..713ae39e7 100644 --- a/pyth/tests/qset/20.result +++ b/pyth/tests/qset/20.result @@ -1 +1 @@ -{"exponent":-3,"price":11037,"conf":271,"status":"trading"} +{"exponent":-3,"price":11000,"conf":2000,"status":"trading"} diff --git a/pyth/tests/qset/21.result b/pyth/tests/qset/21.result index 7ea4eb462..7b68e030f 100644 --- a/pyth/tests/qset/21.result +++ b/pyth/tests/qset/21.result @@ -1 +1 @@ -{"exponent":-3,"price":11101,"conf":963,"status":"trading"} +{"exponent":-3,"price":11100,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/22.result b/pyth/tests/qset/22.result index 92503a1b1..1d9610c90 100644 --- a/pyth/tests/qset/22.result +++ b/pyth/tests/qset/22.result @@ -1 +1 @@ -{"exponent":-3,"price":10000000,"conf":2417,"status":"trading"} +{"exponent":-3,"price":10001000,"conf":4000,"status":"trading"} diff --git a/pyth/tests/qset/24.result b/pyth/tests/qset/24.result index 019f45e19..9abdad927 100644 --- a/pyth/tests/qset/24.result +++ b/pyth/tests/qset/24.result @@ -1 +1 @@ -{"exponent":0,"price":100499,"conf":500,"status":"trading"} +{"exponent":0,"price":100010,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/25.result b/pyth/tests/qset/25.result index 9504c4e09..9abdad927 100644 --- a/pyth/tests/qset/25.result +++ b/pyth/tests/qset/25.result @@ -1 +1 @@ -{"exponent":0,"price":100495,"conf":500,"status":"trading"} +{"exponent":0,"price":100010,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/26.result b/pyth/tests/qset/26.result index 5211511c7..9abdad927 100644 --- a/pyth/tests/qset/26.result +++ b/pyth/tests/qset/26.result @@ -1 +1 @@ -{"exponent":0,"price":100000,"conf":500,"status":"trading"} +{"exponent":0,"price":100010,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/27.result b/pyth/tests/qset/27.result index 019f45e19..a27dd9957 100644 --- a/pyth/tests/qset/27.result +++ b/pyth/tests/qset/27.result @@ -1 +1 @@ -{"exponent":0,"price":100499,"conf":500,"status":"trading"} +{"exponent":0,"price":100500,"conf":500,"status":"trading"} diff --git a/pyth/tests/qset/28.result b/pyth/tests/qset/28.result index 3f20313ea..de26bfeb3 100644 --- a/pyth/tests/qset/28.result +++ b/pyth/tests/qset/28.result @@ -1 +1 @@ -{"exponent":0,"price":101000,"conf":500,"status":"trading"} +{"exponent":0,"price":100990,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/29.result b/pyth/tests/qset/29.result index a27dd9957..de26bfeb3 100644 --- a/pyth/tests/qset/29.result +++ b/pyth/tests/qset/29.result @@ -1 +1 @@ -{"exponent":0,"price":100500,"conf":500,"status":"trading"} +{"exponent":0,"price":100990,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/30.result b/pyth/tests/qset/30.result index a27dd9957..de26bfeb3 100644 --- a/pyth/tests/qset/30.result +++ b/pyth/tests/qset/30.result @@ -1 +1 @@ -{"exponent":0,"price":100500,"conf":500,"status":"trading"} +{"exponent":0,"price":100990,"conf":990,"status":"trading"} diff --git a/pyth/tests/qset/31.result b/pyth/tests/qset/31.result index 41bfa1e6f..fcc8da742 100644 --- a/pyth/tests/qset/31.result +++ b/pyth/tests/qset/31.result @@ -1 +1 @@ -{"exponent":-8,"price":4329847900000,"conf":1758750000,"status":"trading"} +{"exponent":-8,"price":4329187000000,"conf":3187000000,"status":"trading"} diff --git a/pyth/tests/qset/32.result b/pyth/tests/qset/32.result index 41bfa1e6f..19f2993be 100644 --- a/pyth/tests/qset/32.result +++ b/pyth/tests/qset/32.result @@ -1 +1 @@ -{"exponent":-8,"price":4329847900000,"conf":1758750000,"status":"trading"} +{"exponent":-8,"price":4329402176469,"conf":2206823531,"status":"trading"} diff --git a/pyth/tests/qset/33.result b/pyth/tests/qset/33.result index 41bfa1e6f..fcc8da742 100644 --- a/pyth/tests/qset/33.result +++ b/pyth/tests/qset/33.result @@ -1 +1 @@ -{"exponent":-8,"price":4329847900000,"conf":1758750000,"status":"trading"} +{"exponent":-8,"price":4329187000000,"conf":3187000000,"status":"trading"} diff --git a/pyth/tests/qset/34.result b/pyth/tests/qset/34.result index e905d1c69..0df7ca1f6 100644 --- a/pyth/tests/qset/34.result +++ b/pyth/tests/qset/34.result @@ -1 +1 @@ -{"exponent":-8,"price":4290171000000,"conf":59750000,"status":"trading"} +{"exponent":-8,"price":4290170000000,"conf":670275519,"status":"trading"} diff --git a/pyth/tests/qset/35.result b/pyth/tests/qset/35.result index d3f44f001..a54635869 100644 --- a/pyth/tests/qset/35.result +++ b/pyth/tests/qset/35.result @@ -1 +1 @@ -{"exponent":-8,"price":4000200000000,"conf":1000000000,"status":"trading"} +{"exponent":-8,"price":3999400000000,"conf":3959390000000,"status":"trading"} diff --git a/pyth/tests/qset/37.result b/pyth/tests/qset/37.result index d3f44f001..1035783ed 100644 --- a/pyth/tests/qset/37.result +++ b/pyth/tests/qset/37.result @@ -1 +1 @@ -{"exponent":-8,"price":4000200000000,"conf":1000000000,"status":"trading"} +{"exponent":-8,"price":4001000000000,"conf":395998999990000,"status":"trading"} diff --git a/pyth/tests/qset/4.result b/pyth/tests/qset/4.result index c02ca5b6f..76de76f65 100644 --- a/pyth/tests/qset/4.result +++ b/pyth/tests/qset/4.result @@ -1 +1 @@ -{"exponent":-3,"price":10500,"conf":1000,"status":"trading"} +{"exponent":-3,"price":10500,"conf":500,"status":"trading"} diff --git a/pyth/tests/qset/5.result b/pyth/tests/qset/5.result index 5494ac90d..82d896001 100644 --- a/pyth/tests/qset/5.result +++ b/pyth/tests/qset/5.result @@ -1 +1 @@ -{"exponent":-3,"price":15000,"conf":3750,"status":"trading"} +{"exponent":-3,"price":15000,"conf":4000,"status":"trading"} diff --git a/pyth/tests/qset/6.result b/pyth/tests/qset/6.result index 5494ac90d..82d896001 100644 --- a/pyth/tests/qset/6.result +++ b/pyth/tests/qset/6.result @@ -1 +1 @@ -{"exponent":-3,"price":15000,"conf":3750,"status":"trading"} +{"exponent":-3,"price":15000,"conf":4000,"status":"trading"} diff --git a/pyth/tests/qset/7.result b/pyth/tests/qset/7.result index c49bcc780..9e52b037c 100644 --- a/pyth/tests/qset/7.result +++ b/pyth/tests/qset/7.result @@ -1 +1 @@ -{"exponent":-3,"price":100000,"conf":10,"status":"trading"} +{"exponent":-3,"price":100010,"conf":899980,"status":"trading"} diff --git a/pyth/tests/qset/8.result b/pyth/tests/qset/8.result index 99c72f035..b6edfd7f2 100644 --- a/pyth/tests/qset/8.result +++ b/pyth/tests/qset/8.result @@ -1 +1 @@ -{"exponent":-3,"price":10010,"conf":31,"status":"trading"} +{"exponent":-3,"price":10020,"conf":970,"status":"trading"} From 2a1dee64782454f258d1ab590704b586b75dd230 Mon Sep 17 00:00:00 2001 From: Reisen Date: Wed, 23 Feb 2022 12:35:17 +0000 Subject: [PATCH 19/21] oracle: remove unused sort implementation --- program/src/oracle/sort/tmpl/sort_stable.c | 32 ---------------------- 1 file changed, 32 deletions(-) diff --git a/program/src/oracle/sort/tmpl/sort_stable.c b/program/src/oracle/sort/tmpl/sort_stable.c index 9f95e5be7..c42038551 100644 --- a/program/src/oracle/sort/tmpl/sort_stable.c +++ b/program/src/oracle/sort/tmpl/sort_stable.c @@ -173,38 +173,6 @@ SORT_IMPL(stable_node)( SORT_KEY_T * x, } } -# if 0 /* These variants of the above don't seem to produce much better code */ - do { /* Single loop exit */ - if( SORT_BEFORE( yr[k], yl[j] ) ) x[i++] = yr[k++]; - else x[i++] = yl[j++]; - } while( (j Date: Fri, 25 Feb 2022 13:15:44 +0000 Subject: [PATCH 20/21] oracle: add comment to sort explaining network principle --- program/src/oracle/sort/sort_stable_base_gen.c | 3 +++ program/src/oracle/sort/tmpl/sort_stable_base.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/program/src/oracle/sort/sort_stable_base_gen.c b/program/src/oracle/sort/sort_stable_base_gen.c index 70ac61597..598f85a4d 100644 --- a/program/src/oracle/sort/sort_stable_base_gen.c +++ b/program/src/oracle/sort/sort_stable_base_gen.c @@ -56,6 +56,9 @@ sort_gen( int n ) { //printf( " ulong n ) { /* assumes n in [0,%i) */\n", n ); printf( " do { /* BEGIN AUTOGENERATED CODE (n=%2i) *****************************/\n", n ); + printf( " /* This network of comparators and fallthroughs implement a sorting network representation\n" ); + printf( " of an insertion sort. Each case acts as a sort pass with the fallthrough falling through\n" ); + printf( " to smaller ranges of the input. */\n"); printf( "# define SORT_STABLE_CE(i,j) u = x[(SORT_IDX_T)i]; v = x[(SORT_IDX_T)j]; c = SORT_BEFORE( v, u ); x[(SORT_IDX_T)i] = c ? v : u; x[(SORT_IDX_T)j] = c ? u : v\n" ); printf( " int c;\n" ); printf( " SORT_KEY_T u;\n" ); diff --git a/program/src/oracle/sort/tmpl/sort_stable_base.c b/program/src/oracle/sort/tmpl/sort_stable_base.c index 3fc580827..7bc9ce06c 100644 --- a/program/src/oracle/sort/tmpl/sort_stable_base.c +++ b/program/src/oracle/sort/tmpl/sort_stable_base.c @@ -1,4 +1,7 @@ do { /* BEGIN AUTOGENERATED CODE (n= 7) *****************************/ + /* This network of comparators and fallthroughs implement a sorting network representation + of an insertion sort. Each case acts as a sort pass with the fallthrough falling through + to smaller ranges of the input. */ # define SORT_STABLE_CE(i,j) u = x[(SORT_IDX_T)i]; v = x[(SORT_IDX_T)j]; c = SORT_BEFORE( v, u ); x[(SORT_IDX_T)i] = c ? v : u; x[(SORT_IDX_T)j] = c ? u : v int c; SORT_KEY_T u; From 119962c6c5b0ee50c8e0adaf7c2071520354b456 Mon Sep 17 00:00:00 2001 From: Reisen Date: Thu, 3 Mar 2022 14:48:53 +0000 Subject: [PATCH 21/21] oracle: remove old aggregation logic --- program/src/oracle/upd_aggregate.h | 216 +---------------------------- pyth/tests/qset/36.result | 2 +- pyth/tests/qset/39.result | 2 +- 3 files changed, 8 insertions(+), 212 deletions(-) diff --git a/program/src/oracle/upd_aggregate.h b/program/src/oracle/upd_aggregate.h index cd0f79bf8..ed0b8f9ee 100644 --- a/program/src/oracle/upd_aggregate.h +++ b/program/src/oracle/upd_aggregate.h @@ -141,42 +141,6 @@ static inline void upd_twap( upd_ema( &ptr->twac_, conf, conf, nslots, qs ); } -// compute weighted percentile -static void wgt_ptile( - pd_t * const res - , const pd_t * const prices, const pd_t * const weights, const uint32_t num - , const pd_t * const ptile, pc_qset_t *qs ) -{ - pd_t cumwgt[ PC_COMP_SIZE ]; - - pd_t cumwgta[ 1 ], half[ 1 ]; - pd_new( cumwgta, 0, 0 ); - pd_new( half, 5, -1 ); - for ( uint32_t i = 0; i < num; ++i ) { - const pd_t * const wptr = &weights[ i ]; - pd_t weight[ 1 ]; - pd_mul( weight, wptr, half ); - pd_add( &cumwgt[ i ], cumwgta, weight, qs->fact_ ); - pd_add( cumwgta, cumwgta, wptr, qs->fact_ ); - } - - uint32_t i = 0; - for( ; i != num && pd_lt( &cumwgt[i], ptile, qs->fact_ ); ++i ); - if ( i == num ) { - pd_set( res, &prices[num-1] ); - } else if ( i == 0 ) { - pd_set( res, &prices[0] ); - } else { - pd_t t1[1], t2[1]; - pd_sub( t1, &prices[i], &prices[i-1], qs->fact_ ); - pd_sub( t2, ptile, &cumwgt[i-1], qs->fact_ ); - pd_mul( t1, t1, t2 ); - pd_sub( t2, &cumwgt[i], &cumwgt[i-1], qs->fact_ ); - pd_div( t1, t1, t2 ); - pd_add( res, &prices[i-1], t1, qs->fact_ ); - } -} - // update aggregate price static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timestamp ) { @@ -204,11 +168,10 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest // identify valid quotes // compute the aggregate prices and ranges - uint32_t numv = 0; - uint32_t vidx[ PC_COMP_SIZE ]; int64_t agg_price; int64_t agg_conf; { + uint32_t numv = 0; uint32_t nprcs = (uint32_t)0; int64_t prcs[ PC_COMP_SIZE * 3 ]; // ~0.75KiB for current PC_COMP_SIZE (FIXME: DOUBLE CHECK THIS FITS INTO STACK FRAME LIMIT) for ( uint32_t i = 0; i != ptr->num_; ++i ) { @@ -222,7 +185,7 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest if ( iptr->agg_.status_ == PC_STATUS_TRADING && (int64_t)0 < conf && conf < price && conf <= (INT64_MAX-price) && // No overflow for INT64_MAX-price as price>0 slot_diff >= 0 && slot_diff <= PC_MAX_SEND_LATENCY ) { - vidx[numv++] = i; // FIXME: GRAFT FOR OLD PRICE MODEL + numv += 1; prcs[ nprcs++ ] = price - conf; // No overflow as 0 < conf < price prcs[ nprcs++ ] = price; prcs[ nprcs++ ] = price + conf; // No overflow as 0 < conf <= INT64_MAX-price @@ -230,7 +193,7 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest } // too few valid quotes - ptr->num_qt_ = numv; // FIXME: TEMPORARY GRAFT (ALL RETURN PATHS SET NUM_QT TO SOMETHING SENSIBLE) + ptr->num_qt_ = numv; if ( numv == 0 || numv < ptr->min_pub_ ) { ptr->agg_.status_ = PC_STATUS_UNKNOWN; return; @@ -262,180 +225,13 @@ static inline void upd_aggregate( pc_price_t *ptr, uint64_t slot, int64_t timest } } - // FIXME: MOST OF THE REST OF THE CODE HERE CAN PROBABLY BE REMOVED / - // CONDENSED AND/OR MADE MORE EFFICIENT. AND IT PROBABLY WILL - // REPLACED SOON BY THE VWAP MODEL UNDER DEVELOPMENT. IT IS KEPT HERE - // TO KEEP THE CURRENT VWAP CALCULATION AS CLOSE AS POSSIBLE TO WHAT - // HOW IT CURRENTLY WORKS. - - uint32_t numa = 0; - uint32_t aidx[PC_COMP_SIZE]; - { - int64_t const mprc = agg_price; // FIXME: GRAFT TO NEW CALC TO OLD CALC - int64_t const lb = mprc / 5; - int64_t const ub = ( mprc > LONG_MAX / 5 ) ? LONG_MAX : mprc * 5; - - for ( uint32_t i = 0; i < numv; ++i ) { - uint32_t const idx = vidx[ i ]; - pc_price_comp_t const *iptr = &ptr->comp_[ idx ]; - int64_t const prc = iptr->agg_.price_; - if ( prc >= lb && prc <= ub ) { - uint32_t j = numa++; - for( ; j > 0 && ptr->comp_[ aidx[ j - 1 ] ].agg_.price_ > prc; --j ) { // FIXME: O(N^2) - aidx[ j ] = aidx[ j - 1 ]; - } - aidx[ j ] = idx; - } - } - } - - // zero quoters - ptr->num_qt_ = numa; - if ( numa == 0 || numa < ptr->min_pub_ || numa * 2 <= numv ) { - ptr->agg_.status_ = PC_STATUS_UNKNOWN; - return; - } - // update status and publish slot of last trading status price ptr->agg_.status_ = PC_STATUS_TRADING; - ptr->last_slot_ = slot; - - // single quoter - if ( numa == 1 ) { - pc_price_comp_t *iptr = &ptr->comp_[aidx[0]]; - ptr->agg_.price_ = iptr->agg_.price_; - ptr->agg_.conf_ = iptr->agg_.conf_; - upd_twap( ptr, agg_diff, qs ); - ptr->agg_.price_ = agg_price; // FIXME: OVERRIDE OLD PRICE MODEL - ptr->agg_.conf_ = (uint64_t)agg_conf; // FIXME: OVERRIDE OLD PRICE MODEL - return; - } - - // assign quotes and compute weights - pc_price_comp_t *pptr = 0; - pd_t price[1], conf[1], weight[1], one[1], wsum[1]; - pd_new( one, 100000000L, -8 ); - pd_new( wsum, 0, 0 ); - int64_t ldiff = INT64_MAX; - pd_t *wptr = qs->weight_; - for( uint32_t i=0;i != numa; ++i ) { - pc_price_comp_t *iptr = &ptr->comp_[aidx[i]]; - // scale confidence interval by sqrt of slot age - int64_t slot_diff = ( int64_t )slot - ( int64_t )( iptr->agg_.pub_slot_ ); - pd_t decay[1]; - pd_new( decay, qs->decay_[slot_diff], PC_EXP_DECAY ); - pd_new_scale( conf, ( int64_t )( iptr->agg_.conf_ ), ptr->expo_ ); - pd_mul( conf, conf, decay ); - - // assign price and upper/lower bounds of price - pd_new_scale( price, iptr->agg_.price_, ptr->expo_ ); - pd_set( &qs->iprice_[i], price ); - pd_add( &qs->uprice_[i], price, conf, qs->fact_ ); - pd_sub( &qs->lprice_[i], price, conf, qs->fact_ ); - - // compute weight (1/(conf+nearest_neighbor)) - pd_set( &qs->weight_[i], conf ); - if ( i ) { - int64_t idiff = iptr->agg_.price_ - pptr->agg_.price_; - pd_new_scale( weight, idiff < ldiff ? idiff : ldiff, ptr->expo_ ); - pd_add( wptr, wptr, weight, qs->fact_ ); - pd_div( wptr, one, wptr ); - pd_add( wsum, wsum, wptr, qs->fact_ ); - ldiff = idiff; - ++wptr; - } - pptr = iptr; - } - // compute weight for last quote - pd_new_scale( weight, ldiff, ptr->expo_ ); - pd_add( wptr, wptr, weight, qs->fact_ ); - pd_div( wptr, one, wptr ); - pd_add( wsum, wsum, wptr, qs->fact_ ); - - // bound weights at 1/sqrt(Nquotes) and redeistribute the remaining weight - // among the remaining quoters proportional to their weights - pd_t wmax[1], rnumer[1], rdenom[1]; - pd_set( rnumer, one ); - pd_new( rdenom, 0, 0 ); - // wmax = 1 / sqrt( numa ) - pd_new( wmax, numa, 0 ); - pd_sqrt( wmax, wmax, qs->fact_ ); - pd_div( wmax, one, wmax ); - for( uint32_t i=0;i != numa; ++i ) { - wptr = &qs->weight_[i]; - pd_div( wptr, wptr, wsum ); - if ( pd_gt( wptr, wmax, qs->fact_ ) ) { - pd_set( wptr, wmax ); - pd_sub( rnumer, rnumer, wmax, qs->fact_ ); - aidx[i] = 1; - } else { - pd_add( rdenom, rdenom, wptr, qs->fact_ ); - aidx[i] = 0; - } - } - if ( rdenom->v_ ) { - pd_div( rnumer, rnumer, rdenom ); - } - for ( uint32_t i = 0; i != numa; ++i ) { - wptr = &qs->weight_[ i ]; - if ( aidx[ i ] == 0 ) { - pd_mul( wptr, wptr, rnumer ); - } - } - - const pd_t half = { .e_ = -1, .v_ = 5 }; - - // compute aggregate price as weighted median - pd_t iprice[1], lprice[1], uprice[1], q3price[1], q1price[1], ptile[1]; - pd_new( ptile, 5, -1 ); - wgt_ptile( iprice, qs->iprice_, qs->weight_, numa, ptile, qs ); - pd_adjust( iprice, ptr->expo_, qs->fact_ ); - ptr->agg_.price_ = iprice->v_; - - // compute diff in weighted median between upper and lower price bounds - pd_t prices[ PC_COMP_SIZE ]; - pd_t weights[ PC_COMP_SIZE ]; - // sort upper prices and weights - for ( uint32_t i = 0; i < numa; ++i ) { - uint32_t j = i; - for ( ; j > 0 && pd_lt( &qs->uprice_[ i ], &prices[ j - 1 ], qs->fact_ ); --j ) { // FIXME: O(N^2) - prices[ j ] = prices[ j - 1 ]; - weights[ j ] = weights[ j - 1 ]; - } - prices[ j ] = qs->uprice_[ i ]; - weights[ j ] = qs->weight_[ i ]; - } - wgt_ptile( uprice, prices, weights, numa, ptile, qs ); - // sort lower prices and weights - for ( uint32_t i = 0; i < numa; ++i ) { - uint32_t j = i; - for ( ; j > 0 && pd_lt( &qs->lprice_[ i ], &prices[ j - 1 ], qs->fact_ ); --j ) { // FIXME: O(N^2) - prices[ j ] = prices[ j - 1 ]; - weights[ j ] = weights[ j - 1 ]; - } - prices[ j ] = qs->lprice_[ i ]; - weights[ j ] = qs->weight_[ i ]; - } - wgt_ptile( lprice, prices, weights, numa, ptile, qs ); - - pd_sub( uprice, uprice, lprice, qs->fact_ ); - pd_mul( uprice, uprice, &half ); - - // compute weighted iqr of prices - pd_new( ptile, 75, -2 ); - wgt_ptile( q3price, qs->iprice_, qs->weight_, numa, ptile, qs ); - pd_new( ptile, 25, -2 ); - wgt_ptile( q1price, qs->iprice_, qs->weight_, numa, ptile, qs ); - pd_sub( q3price, q3price, q1price, qs->fact_ ); - pd_mul( q3price, q3price, &half ); + ptr->last_slot_ = slot; + ptr->agg_.price_ = agg_price; + ptr->agg_.conf_ = (uint64_t)agg_conf; - // take confidence interval as larger - pd_t *cptr = pd_gt( uprice, q3price, qs->fact_ ) ? uprice : q3price; - pd_adjust( cptr, ptr->expo_, qs->fact_ ); - ptr->agg_.conf_ = ( uint64_t )( cptr->v_ ); upd_twap( ptr, agg_diff, qs ); - ptr->agg_.price_ = agg_price; // FIXME: OVERRIDE OLD PRICE MODEL - ptr->agg_.conf_ = (uint64_t)agg_conf; // FIXME: OVERRIDE OLD PRICE MODEL } #ifdef __cplusplus diff --git a/pyth/tests/qset/36.result b/pyth/tests/qset/36.result index 7d8729c69..f9908b0ff 100644 --- a/pyth/tests/qset/36.result +++ b/pyth/tests/qset/36.result @@ -1 +1 @@ -{"exponent":-8,"price":0,"conf":0,"status":"unknown"} +{"exponent":-8,"price":1002000000,"conf":48998000000,"status":"trading"} diff --git a/pyth/tests/qset/39.result b/pyth/tests/qset/39.result index 7d8729c69..5713dfdd3 100644 --- a/pyth/tests/qset/39.result +++ b/pyth/tests/qset/39.result @@ -1 +1 @@ -{"exponent":-8,"price":0,"conf":0,"status":"unknown"} +{"exponent":-8,"price":115000000,"conf":94900000,"status":"trading"}