diff --git a/third_party/pyth/price-service/.env.sample b/third_party/pyth/price-service/.env.sample index beaee791b3..fed4e7e746 100644 --- a/third_party/pyth/price-service/.env.sample +++ b/third_party/pyth/price-service/.env.sample @@ -17,3 +17,6 @@ PROM_PORT=8081 # The default is to log with level info. #LOG_LEVEL=debug + +REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS=60 +CACHE_TTL_SECONDS=300 diff --git a/third_party/pyth/price-service/docker-compose.mainnet.yaml b/third_party/pyth/price-service/docker-compose.mainnet.yaml index 736a48a3b9..669d6693f0 100644 --- a/third_party/pyth/price-service/docker-compose.mainnet.yaml +++ b/third_party/pyth/price-service/docker-compose.mainnet.yaml @@ -37,6 +37,10 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning + DB_API_CLUSTER: pythnet + REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" + CACHE_TTL_SECONDS: "300" + DB_API_ENDPOINT: "https://web-api.pyth.network" healthcheck: test: [ diff --git a/third_party/pyth/price-service/docker-compose.testnet.yaml b/third_party/pyth/price-service/docker-compose.testnet.yaml index 296ebbe14c..0fb189219c 100644 --- a/third_party/pyth/price-service/docker-compose.testnet.yaml +++ b/third_party/pyth/price-service/docker-compose.testnet.yaml @@ -37,6 +37,10 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning + DB_API_CLUSTER: devnet + REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" + CACHE_TTL_SECONDS: "300" + DB_API_ENDPOINT: "https://web-api.pyth.network" healthcheck: test: [ diff --git a/third_party/pyth/price-service/package-lock.json b/third_party/pyth/price-service/package-lock.json index 2cd8ef60d8..8b412016e7 100644 --- a/third_party/pyth/price-service/package-lock.json +++ b/third_party/pyth/price-service/package-lock.json @@ -1,12 +1,12 @@ { "name": "@pythnetwork/pyth-price-service", - "version": "2.2.4", + "version": "2.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@pythnetwork/pyth-price-service", - "version": "2.2.4", + "version": "2.3.0", "license": "Apache-2.0", "dependencies": { "@certusone/wormhole-sdk": "^0.1.4", @@ -27,6 +27,7 @@ "joi": "^17.6.0", "lru-cache": "^7.14.1", "morgan": "^1.10.0", + "node-fetch": "^2.6.1", "prom-client": "^14.0.1", "response-time": "^2.3.2", "winston": "^3.3.3", @@ -36,6 +37,7 @@ "@types/jest": "^27.5.0", "@types/long": "^4.0.1", "@types/node": "^16.6.1", + "@types/node-fetch": "^2.6.2", "@types/supertest": "^2.0.12", "jest": "^28.0.3", "prettier": "^2.3.2", @@ -634,6 +636,14 @@ "rxjs": "^7.3.0" } }, + "node_modules/@certusone/wormhole-sdk/node_modules/axios": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", + "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@certusone/wormhole-spydk": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@certusone/wormhole-spydk/-/wormhole-spydk-0.0.1.tgz", @@ -2908,6 +2918,30 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.27.tgz", "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, + "node_modules/@types/node-fetch/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/@types/prettier": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz", @@ -3242,14 +3276,6 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "dev": true }, - "node_modules/axios": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", - "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", - "dependencies": { - "follow-redirects": "^1.14.4" - } - }, "node_modules/babel-jest": { "version": "28.0.3", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.0.3.tgz", @@ -4033,6 +4059,25 @@ "node-fetch": "2.6.7" } }, + "node_modules/cross-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -4567,9 +4612,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.14.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", - "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==", + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", "funding": [ { "type": "individual", @@ -7241,22 +7286,11 @@ "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", "engines": { "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } } }, "node_modules/node-gyp-build": { @@ -8589,7 +8623,7 @@ "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "node_modules/triple-beam": { "version": "1.3.0", @@ -8877,12 +8911,12 @@ "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -9483,6 +9517,16 @@ "js-base64": "^3.6.1", "protobufjs": "^6.11.2", "rxjs": "^7.3.0" + }, + "dependencies": { + "axios": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", + "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", + "requires": { + "follow-redirects": "^1.14.4" + } + } } }, "@certusone/wormhole-spydk": { @@ -11124,6 +11168,29 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.27.tgz", "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw==" }, + "@types/node-fetch": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + }, + "dependencies": { + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "@types/prettier": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz", @@ -11430,14 +11497,6 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "dev": true }, - "axios": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", - "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", - "requires": { - "follow-redirects": "^1.14.4" - } - }, "babel-jest": { "version": "28.0.3", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.0.3.tgz", @@ -12067,6 +12126,16 @@ "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", "requires": { "node-fetch": "2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "cross-spawn": { @@ -12506,9 +12575,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "follow-redirects": { - "version": "1.14.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", - "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "form-data": { "version": "4.0.0", @@ -14499,12 +14568,9 @@ "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, "node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "requires": { - "whatwg-url": "^5.0.0" - } + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, "node-gyp-build": { "version": "4.4.0", @@ -15491,7 +15557,7 @@ "tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "triple-beam": { "version": "1.3.0", @@ -15692,12 +15758,12 @@ "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "requires": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" diff --git a/third_party/pyth/price-service/package.json b/third_party/pyth/price-service/package.json index c87f2f2393..a5fb3ed23d 100644 --- a/third_party/pyth/price-service/package.json +++ b/third_party/pyth/price-service/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/pyth-price-service", - "version": "2.2.4", + "version": "2.3.0", "description": "Pyth Price Service", "main": "index.js", "scripts": { @@ -18,6 +18,7 @@ "@types/jest": "^27.5.0", "@types/long": "^4.0.1", "@types/node": "^16.6.1", + "@types/node-fetch": "^2.6.2", "@types/supertest": "^2.0.12", "jest": "^28.0.3", "prettier": "^2.3.2", @@ -46,6 +47,7 @@ "joi": "^17.6.0", "lru-cache": "^7.14.1", "morgan": "^1.10.0", + "node-fetch": "^2.6.1", "prom-client": "^14.0.1", "response-time": "^2.3.2", "winston": "^3.3.3", diff --git a/third_party/pyth/price-service/src/__tests__/rest.test.ts b/third_party/pyth/price-service/src/__tests__/rest.test.ts index 80fd22b4bc..b1ed473711 100644 --- a/third_party/pyth/price-service/src/__tests__/rest.test.ts +++ b/third_party/pyth/price-service/src/__tests__/rest.test.ts @@ -2,11 +2,12 @@ import { HexString, Price, PriceFeed } from "@pythnetwork/pyth-sdk-js"; import { Express } from "express"; import { StatusCodes } from "http-status-codes"; import request from "supertest"; -import { PriceInfo, PriceStore } from "../listen"; +import { PriceInfo, PriceStore, VaaCache } from "../listen"; import { RestAPI } from "../rest"; let app: Express; let priceInfoMap: Map; +let vaasCache: VaaCache; function expandTo64Len(id: string): string { return id.repeat(64).substring(0, 64); @@ -56,6 +57,12 @@ beforeAll(async () => { dummyPriceInfoPair(expandTo64Len("3456"), 2, "bad01bad"), dummyPriceInfoPair(expandTo64Len("10101"), 3, "bidbidbid"), ]); + vaasCache = new VaaCache(); + vaasCache.set( + expandTo64Len("abcd"), + 1, + Buffer.from("a1b2c3d4", "hex").toString("base64") + ); const priceInfo: PriceStore = { getLatestPriceInfo: (priceFeedId: string) => { @@ -63,6 +70,9 @@ beforeAll(async () => { }, addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined, getPriceIds: () => new Set(), + getVaa: (vaasCacheKey: string, publishTime: number) => { + return vaasCache.get(vaasCacheKey, publishTime); + }, }; const api = new RestAPI({ port: 8889 }, priceInfo, () => true); diff --git a/third_party/pyth/price-service/src/__tests__/ws.test.ts b/third_party/pyth/price-service/src/__tests__/ws.test.ts index a397a1c907..aad27090f9 100644 --- a/third_party/pyth/price-service/src/__tests__/ws.test.ts +++ b/third_party/pyth/price-service/src/__tests__/ws.test.ts @@ -108,6 +108,7 @@ beforeAll(async () => { getLatestPriceInfo: (_priceFeedId: string) => undefined, addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined, getPriceIds: () => new Set(priceInfos.map((info) => info.priceFeed.id)), + getVaa: (_vaasCacheKey: string) => undefined, }; api = new WebSocketAPI(priceInfo); diff --git a/third_party/pyth/price-service/src/index.ts b/third_party/pyth/price-service/src/index.ts index 8e97cec1f8..fcee332d42 100644 --- a/third_party/pyth/price-service/src/index.ts +++ b/third_party/pyth/price-service/src/index.ts @@ -52,6 +52,8 @@ async function run() { const restAPI = new RestAPI( { port: parseInt(envOrErr("REST_PORT"), 10), + dbApiEndpoint: process.env.DB_API_ENDPOINT, + dbApiCluster: process.env.DB_API_CLUSTER, }, listener, isReady, @@ -61,6 +63,7 @@ async function run() { const wsAPI = new WebSocketAPI(listener, promClient); listener.run(); + listener.runCacheCleanupLoop(); const server = await restAPI.run(); wsAPI.run(server); } diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 85ba3fc12f..ebdebe2cdd 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -7,21 +7,21 @@ import { import { importCoreWasm } from "@certusone/wormhole-sdk/lib/cjs/solana/wasm"; -import { - getBatchSummary, - parseBatchPriceAttestation, - priceAttestationToPriceFeed, -} from "@pythnetwork/p2w-sdk-js"; import { FilterEntry, SubscribeSignedVAAResponse, } from "@certusone/wormhole-spydk/lib/cjs/proto/spy/v1/spy"; import { ClientReadableStream } from "@grpc/grpc-js"; +import { + getBatchSummary, + parseBatchPriceAttestation, + priceAttestationToPriceFeed, +} from "@pythnetwork/p2w-sdk-js"; import { HexString, PriceFeed } from "@pythnetwork/pyth-sdk-js"; +import LRUCache from "lru-cache"; import { sleep, TimestampInSec } from "./helpers"; import { logger } from "./logging"; import { PromClient } from "./promClient"; -import LRUCache from "lru-cache"; export type PriceInfo = { vaa: Buffer; @@ -37,6 +37,7 @@ export interface PriceStore { getPriceIds(): Set; getLatestPriceInfo(priceFeedId: HexString): PriceInfo | undefined; addUpdateListener(callback: (priceInfo: PriceInfo) => any): void; + getVaa(priceFeedId: string, publishTime: number): VaaConfig | undefined; } type ListenerReadinessConfig = { @@ -48,10 +49,74 @@ type ListenerConfig = { spyServiceHost: string; filtersRaw?: string; readiness: ListenerReadinessConfig; + webApiEndpoint?: string; + webApiCluster?: string; }; type VaaKey = string; +type VaaConfig = { + publishTime: number; + vaa: string; +}; + +export class VaaCache { + private cache: Map; + private ttl: number; + + constructor(ttl: number = 300) { + this.cache = new Map(); + this.ttl = ttl; + } + + set(key: VaaKey, publishTime: number, vaa: string): void { + if (this.cache.has(key)) { + this.cache.get(key)!.push({ publishTime, vaa }); + } else { + this.cache.set(key, [{ publishTime, vaa }]); + } + } + + get(key: VaaKey, publishTime: number): VaaConfig | undefined { + if (!this.cache.has(key)) { + return undefined; + } else { + const vaaConf = this.find(this.cache.get(key)!, publishTime); + return vaaConf; + } + } + + find(arr: VaaConfig[], publishTime: number): VaaConfig | undefined { + if (arr.length === 0 || publishTime < arr[0].publishTime) { + return undefined; + } + let left = 0; + let right = arr.length - 1; + let nextLargest = -1; + + while (left <= right) { + const middle = Math.floor((left + right) / 2); + if (arr[middle].publishTime === publishTime) { + return arr[middle]; + } else if (arr[middle].publishTime < publishTime) { + left = middle + 1; + } else { + nextLargest = middle; + right = middle - 1; + } + } + + return nextLargest !== -1 ? arr[nextLargest] : undefined; + } + + async removeExpiredValues() { + const now = Math.floor(Date.now() / 1000); + for (const arr of this.cache.values()) { + arr.filter((vaaConf) => now - vaaConf.publishTime < this.ttl); + } + } +} + export class Listener implements PriceStore { // Mapping of Price Feed Id to Vaa private priceFeedVaaMap = new Map(); @@ -62,6 +127,7 @@ export class Listener implements PriceStore { private readinessConfig: ListenerReadinessConfig; private updateCallbacks: ((priceInfo: PriceInfo) => any)[]; private observedVaas: LRUCache; + private vaasCache: VaaCache; constructor(config: ListenerConfig, promClient?: PromClient) { this.promClient = promClient; @@ -73,6 +139,7 @@ export class Listener implements PriceStore { max: 10000, // At most 10000 items ttl: 60 * 1000, // 60 seconds }); + this.vaasCache = new VaaCache(); } private loadFilters(filtersRaw?: string) { @@ -105,6 +172,10 @@ export class Listener implements PriceStore { logger.info("loaded " + this.filters.length + " filters"); } + async runCacheCleanupLoop(interval: number = 60) { + setInterval(this.vaasCache.removeExpiredValues, interval * 1000); + } + async run() { logger.info( "pyth_relay starting up, will listen for signed VAAs from " + @@ -185,13 +256,13 @@ export class Listener implements PriceStore { const vaaEmitterAddressHex = Buffer.from( parsedVaa.emitter_address ).toString("hex"); - const vaaKey: VaaKey = `${parsedVaa.emitter_chain}#${vaaEmitterAddressHex}#${parsedVaa.sequence}`; + const observedVaasKey: VaaKey = `${parsedVaa.emitter_chain}#${vaaEmitterAddressHex}#${parsedVaa.sequence}`; - if (this.observedVaas.has(vaaKey)) { + if (this.observedVaas.has(observedVaasKey)) { return; } - this.observedVaas.set(vaaKey, true); + this.observedVaas.set(observedVaasKey, true); this.promClient?.incReceivedVaa(); let batchAttestation; @@ -223,6 +294,11 @@ export class Listener implements PriceStore { const cachedPriceInfo = this.priceFeedVaaMap.get(key); if (this.isNewPriceInfo(cachedPriceInfo, priceInfo)) { + this.vaasCache.set( + priceInfo.priceFeed.id, + priceInfo.publishTime, + priceInfo.vaa.toString("base64") + ); this.priceFeedVaaMap.set(key, priceInfo); if (cachedPriceInfo !== undefined) { @@ -252,6 +328,10 @@ export class Listener implements PriceStore { ); } + getVaa(priceFeedId: string, publishTime: number): VaaConfig | undefined { + return this.vaasCache.get(priceFeedId, publishTime); + } + getLatestPriceInfo(priceFeedId: string): PriceInfo | undefined { return this.priceFeedVaaMap.get(priceFeedId); } diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index 597b8463b1..a751cebf60 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -5,6 +5,7 @@ import { Joi, schema, validate, ValidationError } from "express-validation"; import { Server } from "http"; import { StatusCodes } from "http-status-codes"; import morgan from "morgan"; +import fetch from "node-fetch"; import { TimestampInSec } from "./helpers"; import { PriceStore } from "./listen"; import { logger } from "./logging"; @@ -36,14 +37,18 @@ export class RestAPI { private priceFeedVaaInfo: PriceStore; private isReady: (() => boolean) | undefined; private promClient: PromClient | undefined; + private dbApiEndpoint?: string; + private dbApiCluster?: string; constructor( - config: { port: number }, + config: { port: number; dbApiEndpoint?: string; dbApiCluster?: string }, priceFeedVaaInfo: PriceStore, isReady?: () => boolean, promClient?: PromClient ) { this.port = config.port; + this.dbApiEndpoint = config.dbApiEndpoint; + this.dbApiCluster = config.dbApiCluster; this.priceFeedVaaInfo = priceFeedVaaInfo; this.isReady = isReady; this.promClient = promClient; @@ -113,6 +118,51 @@ export class RestAPI { "api/latest_vaas?ids[]=&ids[]=&.." ); + const getVaaInputSchema: schema = { + query: Joi.object({ + id: Joi.string() + .regex(/^(0x)?[a-f0-9]{64}$/) + .required(), + publish_time: Joi.number().required(), + }).required(), + }; + app.get( + "/api/get_vaa", + validate(getVaaInputSchema), + (req: Request, res: Response) => { + const priceFeedId = req.query.id as string; + const publishTime = Number(req.query.publish_time as string); + const vaa = this.priceFeedVaaInfo.getVaa(priceFeedId, publishTime); + // if publishTime is older than cache ttl or vaa is not found, fetch from db + if (!vaa) { + // cache miss + if (this.dbApiEndpoint && this.dbApiCluster) { + fetch( + `${this.dbApiEndpoint}/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=${this.dbApiCluster}` + ) + .then((r: any) => r.json()) + .then((arr: any) => { + if (arr.length > 0 && arr[0]) { + res.json(arr[0]); + } else { + res.status(StatusCodes.NOT_FOUND).send("VAA not found"); + } + }); + } + } else { + // cache hit + const processedVaa = { + publishTime: new Date(vaa.publishTime), + vaa: vaa.vaa, + }; + res.json(processedVaa); + } + } + ); + endpoints.push( + "api/get_vaa?id=&publish_time=" + ); + const latestPriceFeedsInputSchema: schema = { query: Joi.object({ ids: Joi.array() diff --git a/tilt-devnet/k8s/pyth-price-service.yaml b/tilt-devnet/k8s/pyth-price-service.yaml index 01e7fa6c7f..35cfdcac21 100644 --- a/tilt-devnet/k8s/pyth-price-service.yaml +++ b/tilt-devnet/k8s/pyth-price-service.yaml @@ -72,3 +72,7 @@ spec: value: "6" - name: LOG_LEVEL value: debug + - name: REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS + value: "60" + - name: CACHE_TTL_SECONDS + value: "300"