- new price aggregation and twap algorithms

- reject price updates for same or previous slot
- switch to using programSubscribe instead of accountSubscribe to cut down on websocket subscriptions
- decommission parameter account as new algo doesnt need it
- error handling check for bad pyth_tx connection
- check that previous account is a mapping account when adding new mapping account to list
- fix to add/subtract of numbers with very large diff in magnitude
- track leader slots and tally missing slots by leader from capture withi slots_info and leader_stats binaries
- add -m <commitment> option to control commitment level on program subscription
- rename coalesce to sub_drop since they represent account updates that are dropped by solana subscription
- deserialize price account info per symbol buffer
- hit_rate defined as number received updates where the pub slot changes divided by number of sent transactions
- new callback when price account re-initializes: on_response(price_init*)
- added "pyth get_block" command to to study all pyth transactions in a single block/slot
tighter publish frequency

Author: richardbrooks2000

CMakeLists.txt

@@ -98,8 +98,10 @@ add_executable( test_publish pctest/test_publish.cpp )
 target_link_libraries( test_publish ${PC_DEP} )
 add_executable( test_qset pctest/test_qset.cpp )
 target_link_libraries( test_qset ${PC_DEP} )
-add_executable( test_slot pctest/test_slot.cpp )
-target_link_libraries( test_slot ${PC_DEP} )
+add_executable( slots_info pctest/slots_info.cpp )
+target_link_libraries( slots_info ${PC_DEP} )
+add_executable( leader_stats pctest/leader_stats.cpp )
+target_link_libraries( leader_stats ${PC_DEP} )
 
 add_test( test_unit test_unit )
 add_test( test_net test_net )

dashboard/dashboard.js

@@ -41,7 +41,7 @@ class Prices
     let px   = res.price * this.fact[expo];
     let conf = res.conf * this.fact[expo];
     let twap = res.twap * this.fact[expo];
-    let avol = res.avol * this.fact[expo];
+    let twac = res.twac * this.fact[expo];
     let color = 'cornsilk';
     if ( res.status == 'unknown' ) {
       color = '#c0392b';
@@ -53,7 +53,7 @@ class Prices
     this.draw_cell( row.cells[col++], px.toFixed(expo), color );
     this.draw_cell( row.cells[col++], conf.toFixed(expo), color );
     this.draw_cell( row.cells[col++], twap.toFixed(expo), color );
-    this.draw_cell( row.cells[col++], avol.toFixed(expo), color );
+    this.draw_cell( row.cells[col++], twac.toFixed(expo), color );
     this.draw_cell( row.cells[col++], res.status, color );
     this.draw_cell( row.cells[col++], res.valid_slot, color );
     this.draw_cell( row.cells[col++], res.pub_slot, color );

dashboard/index.html

@@ -18,7 +18,7 @@ <h2>pyth dashboard
           <th width=100px>price</th>
           <th width=100px>conf</th>
           <th width=100px>twap</th>
-          <th width=100px>avol</th>
+          <th width=100px>twac</th>
           <th width=80>status</th>
           <th>valid_slot</th>
           <th>pub_slot</th>

doc/getting_started.md

@@ -29,36 +29,35 @@ This will output the public key in base58 encoding and will look something like:
 5rYvdyWAunZgD2EC1aKo7hQbutUUnkt7bBFM6xNq2z7Z
 ```
 
-Once permissioned, you need three additional public keys in the key-store. The id of the mapping-account that contains the directory listing of the on-chain symbols, the id of the on-chain oracle program that you use to publish prices and the id of the parameter-account that contains various math look-up tables used to compute the aggregate price.  Mapping, program and parameter accounts are maintained in three separate environments: pythnet, devnet and forthcoming mainnet-beta.
+Once permissioned, you need two additional public keys in the key-store. The id of the mapping-account that contains the directory listing of the on-chain symbols and the id of the on-chain oracle program that you use to publish prices.  Mapping and program accounts are maintained in two separate environments: devnet and forthcoming mainnet-beta.
 
 Use the init_key_store.sh script to initialize these account keys:
 
 ```
-KENV=pythnet  # or devnet or mainnet-beta
+KENV=devnet  # or mainnet-beta
 
 # initialize keys for solana devnet
 ../pctest/init_key_store.sh $KENV $KDIR
 
 ```
 
-This creates three files: `$KDIR/mapping_key.json`, `$KDIR/program_key.json`,  and `$KDIR/param_key.json`.
+This creates two files: `$KDIR/mapping_key.json` and `$KDIR/program_key.json`.
 
-Once permissioned, you can test your setup by running the test_publish.cpp example program for publishing and subscribing to two test symbols.  To test publishing on pythnet you need to run:
+Once permissioned, you can test your setup by running the test_publish.cpp example program for publishing and subscribing to two test symbols.  To test publishing on devnet you need to run:
 
 
 ```
-KHOST=44.232.27.44
-./test_publish -k $KDIR -r $KHOST -t $KHOST
+KHOST=api.devnet.solana.com
+THOST=<your_pyth_tx_host>
+./test_publish -k $KDIR -r $KHOST -t $THOST
 ```
 
 The -r and -t options correspond to the locations of required solana validator and pyth_tx server instances.
 
-The certification environment (pythnet) can be found at the IP address 44.232.27.44 where the test validator and pyth_tx instances run. Please also provide to the administrator the IP address you plan to publish to pythnet from so that it can be added to the pythnet firewall.
-
 You can also publish to solana using the pythd server. Start up the server using the same key-store directory and host specification:
 
 ```
-./pythd -k $KDIR -r $KHOST -t $KHOST
+./pythd -k $KDIR -r $KHOST -t $THOST
 ```
 
 Run the test_publish.py example program on the same host to connect to the pythd server:
@@ -76,7 +75,7 @@ The solana rpc interface is used for tracking slot and account updates. The pyth
 
 pyth_tx runs as a separate server rather than being integrated directly into the client library to avoid a publisher from having to submit UDP datagrams to hundreds of different IP address from within an organization's firewall. Instead, a pyth_tx server can be deployed outside the firewall and a publisher clienti, running inside the firewall, can make a single TCP connection to it on a dedicated port.
 
-You can run your own pyth_tx instance (recommended) or connect to provided server instances running against pythnet and mainnet-beta environments.
+You can run your own pyth_tx instance (recommended) or connect to provided server instances running against mainnet-beta.
 
 Start up the pyth_tx server as follows:
 

doc/migrating_to_version_2.md

@@ -41,9 +41,7 @@ key_store::key_store()
   has_mkey_( false ),
   has_mpub_( false ),
   has_gkey_( false ),
-  has_gpub_( false ),
-  has_akey_( false ),
-  has_apub_( false )
+  has_gpub_( false )
 {
 }
 
@@ -128,16 +126,6 @@ std::string key_store::get_program_pub_key_file() const
   return dir_ + "program_key.json";
 }
 
-std::string key_store::get_param_key_pair_file() const
-{
-  return dir_ + "param_key_pair.json";
-}
-
-std::string key_store::get_param_pub_key_file() const
-{
-  return dir_ + "param_key.json";
-}
-
 key_pair *key_store::create_publish_key_pair()
 {
   pkey_.gen();
@@ -265,47 +253,6 @@ pub_key *key_store::get_program_pub_key()
   return &gpub_;
 }
 
-key_pair *key_store::create_param_key_pair()
-{
-  akey_.gen();
-  if ( !write_key_file( get_param_key_pair_file(), akey_ ) ) {
-    return nullptr;
-  }
-  akey_.get_pub_key( apub_ );
-  has_akey_ = true;
-  has_apub_ = true;
-  return &akey_;
-}
-
-key_pair *key_store::get_param_key_pair()
-{
-  if ( has_akey_ ) {
-    return &akey_;
-  }
-  if ( !akey_.init_from_file( get_param_key_pair_file() ) ) {
-    return nullptr;
-  }
-  akey_.get_pub_key( apub_ );
-  has_akey_ = true;
-  has_apub_ = true;
-  return &akey_;
-}
-
-pub_key *key_store::get_param_pub_key()
-{
-  if ( has_apub_ ) {
-    return &apub_;
-  }
-  if ( get_param_key_pair() ) {
-    return &apub_;
-  }
-  if ( !apub_.init_from_file( get_param_pub_key_file() ) ) {
-    return nullptr;
-  }
-  has_apub_ = true;
-  return &apub_;
-}
-
 bool key_store::create_account_key_pair( key_pair& res )
 {
   res.gen();

pc/key_store.cpp

@@ -29,8 +29,6 @@ namespace pc
     std::string get_mapping_pub_key_file() const;
     std::string get_program_key_pair_file() const;
     std::string get_program_pub_key_file() const;
-    std::string get_param_key_pair_file() const;
-    std::string get_param_pub_key_file() const;
 
     // primary publishing and funding key
     key_pair *create_publish_key_pair();
@@ -48,11 +46,6 @@ namespace pc
     key_pair *get_program_key_pair();
     pub_key  *get_program_pub_key();
 
-    // get parameter account key_pair or public key
-    key_pair *create_param_key_pair();
-    key_pair *get_param_key_pair();
-    pub_key  *get_param_pub_key();
-
     // create new mapping or symbol account
     bool create_account_key_pair( key_pair& );
     bool get_account_key_pair( const pub_key&, key_pair& );
@@ -64,17 +57,13 @@ namespace pc
     bool        has_mpub_;
     bool        has_gkey_;
     bool        has_gpub_;
-    bool        has_akey_;
-    bool        has_apub_;
     key_pair    pkey_; // primary publishing and funding key
     key_pair    mkey_; // mapping account key
     key_pair    gkey_; // program key pair
-    key_pair    akey_; // parameter account key pair
     key_cache   ckey_; // publishing cache key
     pub_key     ppub_; // publisher public key
     pub_key     mpub_; // mapping account public key
     pub_key     gpub_; // program id
-    pub_key     apub_; // parameter account public key
     std::string dir_;  // key store directory
   };