From 3926e9fb625cc10dd7590ea459cad0c0bfe7dfea Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 12 Jan 2023 12:15:01 +0000 Subject: [PATCH 01/21] add /get_vaa endpoint that serves cache and db vaas --- .../pyth/price-service/package-lock.json | 162 ++++++++++++------ third_party/pyth/price-service/package.json | 2 + .../price-service/src/__tests__/rest.test.ts | 26 ++- .../price-service/src/__tests__/ws.test.ts | 1 + third_party/pyth/price-service/src/helpers.ts | 2 +- third_party/pyth/price-service/src/index.ts | 1 + third_party/pyth/price-service/src/listen.ts | 108 ++++++++++-- third_party/pyth/price-service/src/rest.ts | 55 +++++- 8 files changed, 294 insertions(+), 63 deletions(-) diff --git a/third_party/pyth/price-service/package-lock.json b/third_party/pyth/price-service/package-lock.json index 2cd8ef60d8..08561800ba 100644 --- a/third_party/pyth/price-service/package-lock.json +++ b/third_party/pyth/price-service/package-lock.json @@ -27,6 +27,7 @@ "joi": "^17.6.0", "lru-cache": "^7.14.1", "morgan": "^1.10.0", + "node-fetch": "^2.6.1", "prom-client": "^14.0.1", "response-time": "^2.3.2", "winston": "^3.3.3", @@ -36,6 +37,7 @@ "@types/jest": "^27.5.0", "@types/long": "^4.0.1", "@types/node": "^16.6.1", + "@types/node-fetch": "^2.6.2", "@types/supertest": "^2.0.12", "jest": "^28.0.3", "prettier": "^2.3.2", @@ -634,6 +636,14 @@ "rxjs": "^7.3.0" } }, + "node_modules/@certusone/wormhole-sdk/node_modules/axios": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", + "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@certusone/wormhole-spydk": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/@certusone/wormhole-spydk/-/wormhole-spydk-0.0.1.tgz", @@ -2908,6 +2918,30 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.27.tgz", "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, + "node_modules/@types/node-fetch/node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/@types/prettier": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz", @@ -3242,14 +3276,6 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "dev": true }, - "node_modules/axios": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", - "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", - "dependencies": { - "follow-redirects": "^1.14.4" - } - }, "node_modules/babel-jest": { "version": "28.0.3", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.0.3.tgz", @@ -4033,6 +4059,25 @@ "node-fetch": "2.6.7" } }, + "node_modules/cross-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -4567,9 +4612,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.14.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", - "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==", + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", "funding": [ { "type": "individual", @@ -7241,22 +7286,11 @@ "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", "engines": { "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } } }, "node_modules/node-gyp-build": { @@ -8589,7 +8623,7 @@ "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "node_modules/triple-beam": { "version": "1.3.0", @@ -8877,12 +8911,12 @@ "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -9483,6 +9517,16 @@ "js-base64": "^3.6.1", "protobufjs": "^6.11.2", "rxjs": "^7.3.0" + }, + "dependencies": { + "axios": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", + "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", + "requires": { + "follow-redirects": "^1.14.4" + } + } } }, "@certusone/wormhole-spydk": { @@ -11124,6 +11168,29 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-16.11.27.tgz", "integrity": "sha512-C1pD3kgLoZ56Uuy5lhfOxie4aZlA3UMGLX9rXteq4WitEZH6Rl80mwactt9QG0w0gLFlN/kLBTFnGXtDVWvWQw==" }, + "@types/node-fetch": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.2.tgz", + "integrity": "sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + }, + "dependencies": { + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "@types/prettier": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.6.0.tgz", @@ -11430,14 +11497,6 @@ "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", "dev": true }, - "axios": { - "version": "0.24.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.24.0.tgz", - "integrity": "sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==", - "requires": { - "follow-redirects": "^1.14.4" - } - }, "babel-jest": { "version": "28.0.3", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-28.0.3.tgz", @@ -12067,6 +12126,16 @@ "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", "requires": { "node-fetch": "2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "cross-spawn": { @@ -12506,9 +12575,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "follow-redirects": { - "version": "1.14.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", - "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "form-data": { "version": "4.0.0", @@ -14499,12 +14568,9 @@ "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, "node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", - "requires": { - "whatwg-url": "^5.0.0" - } + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" }, "node-gyp-build": { "version": "4.4.0", @@ -15491,7 +15557,7 @@ "tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "triple-beam": { "version": "1.3.0", @@ -15692,12 +15758,12 @@ "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "requires": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" diff --git a/third_party/pyth/price-service/package.json b/third_party/pyth/price-service/package.json index c87f2f2393..02bbebf66b 100644 --- a/third_party/pyth/price-service/package.json +++ b/third_party/pyth/price-service/package.json @@ -18,6 +18,7 @@ "@types/jest": "^27.5.0", "@types/long": "^4.0.1", "@types/node": "^16.6.1", + "@types/node-fetch": "^2.6.2", "@types/supertest": "^2.0.12", "jest": "^28.0.3", "prettier": "^2.3.2", @@ -46,6 +47,7 @@ "joi": "^17.6.0", "lru-cache": "^7.14.1", "morgan": "^1.10.0", + "node-fetch": "^2.6.1", "prom-client": "^14.0.1", "response-time": "^2.3.2", "winston": "^3.3.3", diff --git a/third_party/pyth/price-service/src/__tests__/rest.test.ts b/third_party/pyth/price-service/src/__tests__/rest.test.ts index 80fd22b4bc..b03820136a 100644 --- a/third_party/pyth/price-service/src/__tests__/rest.test.ts +++ b/third_party/pyth/price-service/src/__tests__/rest.test.ts @@ -2,11 +2,12 @@ import { HexString, Price, PriceFeed } from "@pythnetwork/pyth-sdk-js"; import { Express } from "express"; import { StatusCodes } from "http-status-codes"; import request from "supertest"; -import { PriceInfo, PriceStore } from "../listen"; +import { PriceInfo, PriceStore, VaaCache } from "../listen"; import { RestAPI } from "../rest"; let app: Express; let priceInfoMap: Map; +let vaasCache: VaaCache; function expandTo64Len(id: string): string { return id.repeat(64).substring(0, 64); @@ -56,6 +57,12 @@ beforeAll(async () => { dummyPriceInfoPair(expandTo64Len("3456"), 2, "bad01bad"), dummyPriceInfoPair(expandTo64Len("10101"), 3, "bidbidbid"), ]); + vaasCache = new VaaCache(); + vaasCache.set( + expandTo64Len("abcd"), + 1, + Buffer.from("a1b2c3d4", "hex").toString("base64") + ); const priceInfo: PriceStore = { getLatestPriceInfo: (priceFeedId: string) => { @@ -63,6 +70,9 @@ beforeAll(async () => { }, addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined, getPriceIds: () => new Set(), + getVaa: (vaasCacheKey: string, publishTime: number) => { + return vaasCache.get(vaasCacheKey, publishTime); + }, }; const api = new RestAPI({ port: 8889 }, priceInfo, () => true); @@ -146,3 +156,17 @@ describe("Latest Vaa Bytes Endpoint", () => { expect(resp.body.message).toContain(ids[2]); }); }); + +// describe("Get Vaa Endpoint", () => { +// test("When called with valid id and publish_time, returns vaa string if cached", async () => { +// const id = expandTo64Len("abcd"); +// const resp = await request(app) +// .get("/api/get_vaa") +// .query({ id, publish_time: 1 }); +// expect(resp.status).toBe(StatusCodes.OK); +// expect(resp.body.length).toBe(8); +// expect(resp.body).toContain( +// Buffer.from("a1b2c3d4", "hex").toString("base64") +// ); +// }); +// }); diff --git a/third_party/pyth/price-service/src/__tests__/ws.test.ts b/third_party/pyth/price-service/src/__tests__/ws.test.ts index a397a1c907..ce9eb63e2d 100644 --- a/third_party/pyth/price-service/src/__tests__/ws.test.ts +++ b/third_party/pyth/price-service/src/__tests__/ws.test.ts @@ -108,6 +108,7 @@ beforeAll(async () => { getLatestPriceInfo: (_priceFeedId: string) => undefined, addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined, getPriceIds: () => new Set(priceInfos.map((info) => info.priceFeed.id)), + getVaa: (_vaasCacheKey: string) => null, }; api = new WebSocketAPI(priceInfo); diff --git a/third_party/pyth/price-service/src/helpers.ts b/third_party/pyth/price-service/src/helpers.ts index 2793327bb4..f7d38eb889 100644 --- a/third_party/pyth/price-service/src/helpers.ts +++ b/third_party/pyth/price-service/src/helpers.ts @@ -14,4 +14,4 @@ export function envOrErr(env: string): string { throw new Error(`environment variable "${env}" must be set`); } return String(process.env[env]); -} +} \ No newline at end of file diff --git a/third_party/pyth/price-service/src/index.ts b/third_party/pyth/price-service/src/index.ts index 8e97cec1f8..77528ffcf6 100644 --- a/third_party/pyth/price-service/src/index.ts +++ b/third_party/pyth/price-service/src/index.ts @@ -61,6 +61,7 @@ async function run() { const wsAPI = new WebSocketAPI(listener, promClient); listener.run(); + listener.removeExpiredValuesFromVaasCache(); const server = await restAPI.run(); wsAPI.run(server); } diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 85ba3fc12f..b0b880dc06 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -2,26 +2,26 @@ import { ChainId, uint8ArrayToHex } from "@certusone/wormhole-sdk"; import { createSpyRPCServiceClient, - subscribeSignedVAA, + subscribeSignedVAA } from "@certusone/wormhole-spydk"; import { importCoreWasm } from "@certusone/wormhole-sdk/lib/cjs/solana/wasm"; -import { - getBatchSummary, - parseBatchPriceAttestation, - priceAttestationToPriceFeed, -} from "@pythnetwork/p2w-sdk-js"; import { FilterEntry, - SubscribeSignedVAAResponse, + SubscribeSignedVAAResponse } from "@certusone/wormhole-spydk/lib/cjs/proto/spy/v1/spy"; import { ClientReadableStream } from "@grpc/grpc-js"; +import { + getBatchSummary, + parseBatchPriceAttestation, + priceAttestationToPriceFeed +} from "@pythnetwork/p2w-sdk-js"; import { HexString, PriceFeed } from "@pythnetwork/pyth-sdk-js"; -import { sleep, TimestampInSec } from "./helpers"; +import LRUCache from "lru-cache"; +import { envOrErr, sleep, TimestampInSec } from "./helpers"; import { logger } from "./logging"; import { PromClient } from "./promClient"; -import LRUCache from "lru-cache"; export type PriceInfo = { vaa: Buffer; @@ -37,6 +37,7 @@ export interface PriceStore { getPriceIds(): Set; getLatestPriceInfo(priceFeedId: HexString): PriceInfo | undefined; addUpdateListener(callback: (priceInfo: PriceInfo) => any): void; + getVaa(priceFeedId: string, publishTime: number): VaaConfig | null; } type ListenerReadinessConfig = { @@ -52,6 +53,67 @@ type ListenerConfig = { type VaaKey = string; +type VaaConfig = { + publishTime: number; + vaa: string; +}; + +export class VaaCache { + private cache: { [key: string]: VaaConfig[] }; + private ttl: number; + + constructor() { + this.cache = {}; + this.ttl = Number(envOrErr("CACHE_TTL_SECONDS")); + } + + set(key: VaaKey, publishTime: number, vaa: string) { + if (this.cache[key]) { + this.cache[key].push({ publishTime, vaa }); + } else { + this.cache[key] = [{ publishTime, vaa }]; + } + } + + get(key: VaaKey, publishTime: number) { + if (!this.cache[key]) { + return null; + } else { + const vaaConf = this.find(this.cache[key], publishTime); + return vaaConf; + } + } + + find(arr: VaaConfig[], publishTime: number) { + let left = 0; + let right = arr.length - 1; + let nextLargest = -1; + + while (left <= right) { + const middle = Math.floor((left + right) / 2); + if (arr[middle].publishTime === publishTime) { + return arr[middle]; + } else if (arr[middle].publishTime < publishTime) { + left = middle + 1; + } else { + nextLargest = middle; + right = middle - 1; + } + } + + return nextLargest !== -1 ? arr[nextLargest] : null; + } + + async removeExpiredValues() { + const now = Math.floor(Date.now() / 1000); + for (const key in this.cache) { + this.cache[key] = this.cache[key].filter( + (vaaConf) => now - vaaConf.publishTime < this.ttl + ); + } + } +} + export class Listener implements PriceStore { // Mapping of Price Feed Id to Vaa private priceFeedVaaMap = new Map(); @@ -62,6 +124,7 @@ export class Listener implements PriceStore { private readinessConfig: ListenerReadinessConfig; private updateCallbacks: ((priceInfo: PriceInfo) => any)[]; private observedVaas: LRUCache; + private vaasCache: VaaCache; constructor(config: ListenerConfig, promClient?: PromClient) { this.promClient = promClient; @@ -73,6 +136,7 @@ export class Listener implements PriceStore { max: 10000, // At most 10000 items ttl: 60 * 1000, // 60 seconds }); + this.vaasCache = new VaaCache(); } private loadFilters(filtersRaw?: string) { @@ -105,6 +169,17 @@ export class Listener implements PriceStore { logger.info("loaded " + this.filters.length + " filters"); } + async removeExpiredValuesFromVaasCache() { + const REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS = Number( + envOrErr("REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS") + ); + // run this every REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS seconds + while (true) { + await this.vaasCache.removeExpiredValues(); + await sleep(REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS * 1000); + } + } + async run() { logger.info( "pyth_relay starting up, will listen for signed VAAs from " + @@ -185,13 +260,13 @@ export class Listener implements PriceStore { const vaaEmitterAddressHex = Buffer.from( parsedVaa.emitter_address ).toString("hex"); - const vaaKey: VaaKey = `${parsedVaa.emitter_chain}#${vaaEmitterAddressHex}#${parsedVaa.sequence}`; + const observedVaasKey: VaaKey = `${parsedVaa.emitter_chain}#${vaaEmitterAddressHex}#${parsedVaa.sequence}`; - if (this.observedVaas.has(vaaKey)) { + if (this.observedVaas.has(observedVaasKey)) { return; } - this.observedVaas.set(vaaKey, true); + this.observedVaas.set(observedVaasKey, true); this.promClient?.incReceivedVaa(); let batchAttestation; @@ -223,6 +298,11 @@ export class Listener implements PriceStore { const cachedPriceInfo = this.priceFeedVaaMap.get(key); if (this.isNewPriceInfo(cachedPriceInfo, priceInfo)) { + this.vaasCache.set( + priceInfo.priceFeed.id, + priceInfo.publishTime, + priceInfo.vaa.toString("base64") + ); this.priceFeedVaaMap.set(key, priceInfo); if (cachedPriceInfo !== undefined) { @@ -252,6 +332,10 @@ export class Listener implements PriceStore { ); } + getVaa(priceFeedId: string, publishTime: number): VaaConfig | null { + return this.vaasCache.get(priceFeedId, publishTime); + } + getLatestPriceInfo(priceFeedId: string): PriceInfo | undefined { return this.priceFeedVaaMap.get(priceFeedId); } diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index 597b8463b1..83f6302c4d 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -1,14 +1,21 @@ import { HexString } from "@pythnetwork/pyth-sdk-js"; import cors from "cors"; +import * as dotenv from "dotenv"; // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import express, { NextFunction, Request, Response } from "express"; import { Joi, schema, validate, ValidationError } from "express-validation"; import { Server } from "http"; import { StatusCodes } from "http-status-codes"; import morgan from "morgan"; -import { TimestampInSec } from "./helpers"; +import fetch from "node-fetch"; +import { envOrErr, TimestampInSec } from "./helpers"; import { PriceStore } from "./listen"; import { logger } from "./logging"; import { PromClient } from "./promClient"; +dotenv.config(); + +const PYTH_WRITER_INSERT_LATENCY_SECONDS = Number( + envOrErr("PYTH_WRITER_INSERT_LATENCY_SECONDS") +); const MORGAN_LOG_FORMAT = ':remote-addr - :remote-user ":method :url HTTP/:http-version"' + @@ -113,6 +120,52 @@ export class RestAPI { "api/latest_vaas?ids[]=&ids[]=&.." ); + const getVaaInputSchema: schema = { + query: Joi.object({ + id: Joi.string() + .regex(/^(0x)?[a-f0-9]{64}$/) + .required(), + // publishTime is of UNIX timestamp type + publish_time: Joi.number().required(), + }).required(), + }; + app.get( + "/api/get_vaa", + validate(getVaaInputSchema), + (req: Request, res: Response) => { + const priceFeedId = req.query.id as string; + const publishTime = Number(req.query.publish_time as string); + const vaa = this.priceFeedVaaInfo.getVaa(priceFeedId, publishTime); + // if publishTime is older than cache ttl or vaa is not found, fetch from db + if ( + publishTime < + Math.floor(Date.now() / 1000) - + PYTH_WRITER_INSERT_LATENCY_SECONDS || + !vaa + ) { + // cache miss + fetch( + `https://web-api.pyth.network/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=pythnet` + ) + .then((r: any) => r.json()) + .then((arr: any) => { + if (arr.length > 0 && arr[0]) { + res.json(arr[0]); + } else { + res.status(404).send("VAA not found"); + } + }); + } else { + // cache hit + const processedVaa = {publishTime: new Date(vaa.publishTime), vaa: vaa.vaa} + res.json(processedVaa); + } + } + ); + endpoints.push( + "api/get_vaa?id=&publish_time=" + ); + const latestPriceFeedsInputSchema: schema = { query: Joi.object({ ids: Joi.array() From 1f4987b05dfe1c7d83a690e14803e0392f687479 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 12 Jan 2023 12:21:51 +0000 Subject: [PATCH 02/21] update env vars --- third_party/pyth/price-service/.env.sample | 5 +++++ third_party/pyth/price-service/docker-compose.mainnet.yaml | 4 ++++ third_party/pyth/price-service/docker-compose.testnet.yaml | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/third_party/pyth/price-service/.env.sample b/third_party/pyth/price-service/.env.sample index beaee791b3..872e392044 100644 --- a/third_party/pyth/price-service/.env.sample +++ b/third_party/pyth/price-service/.env.sample @@ -17,3 +17,8 @@ PROM_PORT=8081 # The default is to log with level info. #LOG_LEVEL=debug + +CLUSTER=pythnet +REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS=60 +PYTH_WRITER_INSERT_LATENCY_SECONDS=30 +CACHE_TTL_SECONDS=300 \ No newline at end of file diff --git a/third_party/pyth/price-service/docker-compose.mainnet.yaml b/third_party/pyth/price-service/docker-compose.mainnet.yaml index 736a48a3b9..c8726e1ac6 100644 --- a/third_party/pyth/price-service/docker-compose.mainnet.yaml +++ b/third_party/pyth/price-service/docker-compose.mainnet.yaml @@ -37,6 +37,10 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning + CLUSTER: pythnet + REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" + PYTH_WRITER_INSERT_LATENCY_SECONDS: "30" + CACHE_TTL_SECONDS: "300" healthcheck: test: [ diff --git a/third_party/pyth/price-service/docker-compose.testnet.yaml b/third_party/pyth/price-service/docker-compose.testnet.yaml index 296ebbe14c..ab49c89d41 100644 --- a/third_party/pyth/price-service/docker-compose.testnet.yaml +++ b/third_party/pyth/price-service/docker-compose.testnet.yaml @@ -37,6 +37,10 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning + CLUSTER: pythtest + REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" + PYTH_WRITER_INSERT_LATENCY_SECONDS: "30" + CACHE_TTL_SECONDS: "300" healthcheck: test: [ From 0b7e400ff6790d01f12c142b2d0b87108c88ee89 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 12 Jan 2023 12:35:34 +0000 Subject: [PATCH 03/21] fix precommit errors --- third_party/pyth/price-service/src/helpers.ts | 2 +- third_party/pyth/price-service/src/listen.ts | 6 +++--- third_party/pyth/price-service/src/rest.ts | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/third_party/pyth/price-service/src/helpers.ts b/third_party/pyth/price-service/src/helpers.ts index f7d38eb889..2793327bb4 100644 --- a/third_party/pyth/price-service/src/helpers.ts +++ b/third_party/pyth/price-service/src/helpers.ts @@ -14,4 +14,4 @@ export function envOrErr(env: string): string { throw new Error(`environment variable "${env}" must be set`); } return String(process.env[env]); -} \ No newline at end of file +} diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index b0b880dc06..0bdef9db7e 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -2,20 +2,20 @@ import { ChainId, uint8ArrayToHex } from "@certusone/wormhole-sdk"; import { createSpyRPCServiceClient, - subscribeSignedVAA + subscribeSignedVAA, } from "@certusone/wormhole-spydk"; import { importCoreWasm } from "@certusone/wormhole-sdk/lib/cjs/solana/wasm"; import { FilterEntry, - SubscribeSignedVAAResponse + SubscribeSignedVAAResponse, } from "@certusone/wormhole-spydk/lib/cjs/proto/spy/v1/spy"; import { ClientReadableStream } from "@grpc/grpc-js"; import { getBatchSummary, parseBatchPriceAttestation, - priceAttestationToPriceFeed + priceAttestationToPriceFeed, } from "@pythnetwork/p2w-sdk-js"; import { HexString, PriceFeed } from "@pythnetwork/pyth-sdk-js"; import LRUCache from "lru-cache"; diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index 83f6302c4d..f77a08f543 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -157,7 +157,10 @@ export class RestAPI { }); } else { // cache hit - const processedVaa = {publishTime: new Date(vaa.publishTime), vaa: vaa.vaa} + const processedVaa = { + publishTime: new Date(vaa.publishTime), + vaa: vaa.vaa, + }; res.json(processedVaa); } } From dc92872b19be8c2c1abed09870566b1b31d8613d Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Thu, 12 Jan 2023 13:16:10 +0000 Subject: [PATCH 04/21] fix precommit errors --- third_party/pyth/price-service/.env.sample | 2 +- third_party/pyth/price-service/src/rest.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/third_party/pyth/price-service/.env.sample b/third_party/pyth/price-service/.env.sample index 872e392044..6a36c5d110 100644 --- a/third_party/pyth/price-service/.env.sample +++ b/third_party/pyth/price-service/.env.sample @@ -21,4 +21,4 @@ PROM_PORT=8081 CLUSTER=pythnet REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS=60 PYTH_WRITER_INSERT_LATENCY_SECONDS=30 -CACHE_TTL_SECONDS=300 \ No newline at end of file +CACHE_TTL_SECONDS=300 diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index f77a08f543..9ac2934ba5 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -125,7 +125,6 @@ export class RestAPI { id: Joi.string() .regex(/^(0x)?[a-f0-9]{64}$/) .required(), - // publishTime is of UNIX timestamp type publish_time: Joi.number().required(), }).required(), }; From a202f222b2b79febfe0ff0d47ae28eeac6d0afe2 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 13:39:25 +0900 Subject: [PATCH 05/21] rename removeExpiredValuesFromVaasCache to runCacheCleanupLoop --- third_party/pyth/price-service/src/index.ts | 2 +- third_party/pyth/price-service/src/listen.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/pyth/price-service/src/index.ts b/third_party/pyth/price-service/src/index.ts index 77528ffcf6..a05ed48a26 100644 --- a/third_party/pyth/price-service/src/index.ts +++ b/third_party/pyth/price-service/src/index.ts @@ -61,7 +61,7 @@ async function run() { const wsAPI = new WebSocketAPI(listener, promClient); listener.run(); - listener.removeExpiredValuesFromVaasCache(); + listener.runCacheCleanupLoop(); const server = await restAPI.run(); wsAPI.run(server); } diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 0bdef9db7e..a479b412be 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -169,7 +169,7 @@ export class Listener implements PriceStore { logger.info("loaded " + this.filters.length + " filters"); } - async removeExpiredValuesFromVaasCache() { + async runCacheCleanupLoop() { const REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS = Number( envOrErr("REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS") ); From 8371d3265ba7132a6320efde970256115d03278c Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:57:37 +0900 Subject: [PATCH 06/21] move initialized envOrErr to constructor or func arg --- third_party/pyth/price-service/src/listen.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index a479b412be..dee7e1dac5 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -19,7 +19,7 @@ import { } from "@pythnetwork/p2w-sdk-js"; import { HexString, PriceFeed } from "@pythnetwork/pyth-sdk-js"; import LRUCache from "lru-cache"; -import { envOrErr, sleep, TimestampInSec } from "./helpers"; +import { sleep, TimestampInSec } from "./helpers"; import { logger } from "./logging"; import { PromClient } from "./promClient"; @@ -62,9 +62,9 @@ export class VaaCache { private cache: { [key: string]: VaaConfig[] }; private ttl: number; - constructor() { + constructor(ttl: number = 300) { this.cache = {}; - this.ttl = Number(envOrErr("CACHE_TTL_SECONDS")); + this.ttl = ttl; } set(key: VaaKey, publishTime: number, vaa: string) { @@ -169,14 +169,11 @@ export class Listener implements PriceStore { logger.info("loaded " + this.filters.length + " filters"); } - async runCacheCleanupLoop() { - const REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS = Number( - envOrErr("REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS") - ); - // run this every REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS seconds + async runCacheCleanupLoop(interval: number = 60) { + // run this every interval seconds while (true) { await this.vaasCache.removeExpiredValues(); - await sleep(REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS * 1000); + await sleep(interval * 1000); } } From 8b95b6bab802e4101952c884cdd2d9f264db4cf0 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 07:48:11 +0000 Subject: [PATCH 07/21] use setInterval --- third_party/pyth/price-service/src/listen.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index dee7e1dac5..63dba0ea93 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -170,11 +170,7 @@ export class Listener implements PriceStore { } async runCacheCleanupLoop(interval: number = 60) { - // run this every interval seconds - while (true) { - await this.vaasCache.removeExpiredValues(); - await sleep(interval * 1000); - } + setInterval(this.vaasCache.removeExpiredValues, interval * 1000) } async run() { From bfae5d67da8a20e431bbecdbbfe220a6cceb8739 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 07:51:36 +0000 Subject: [PATCH 08/21] update pyth-price-service env vars on tilt devnet --- tilt-devnet/k8s/pyth-price-service.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tilt-devnet/k8s/pyth-price-service.yaml b/tilt-devnet/k8s/pyth-price-service.yaml index 01e7fa6c7f..273e91f59a 100644 --- a/tilt-devnet/k8s/pyth-price-service.yaml +++ b/tilt-devnet/k8s/pyth-price-service.yaml @@ -72,3 +72,11 @@ spec: value: "6" - name: LOG_LEVEL value: debug + - name: CLUSTER + value: devnet + - name: REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS + value: "60" + - name: PYTH_WRITER_INSERT_LATENCY_SECONDS + value: "30" + - name: CACHE_TTL_SECONDS + value: "300" \ No newline at end of file From 75494e14da835feb9c948f7860e89ce4e40cf303 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 07:55:25 +0000 Subject: [PATCH 09/21] use undefined instead of null --- third_party/pyth/price-service/src/listen.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 63dba0ea93..b69f84397a 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -37,7 +37,7 @@ export interface PriceStore { getPriceIds(): Set; getLatestPriceInfo(priceFeedId: HexString): PriceInfo | undefined; addUpdateListener(callback: (priceInfo: PriceInfo) => any): void; - getVaa(priceFeedId: string, publishTime: number): VaaConfig | null; + getVaa(priceFeedId: string, publishTime: number): VaaConfig | undefined; } type ListenerReadinessConfig = { @@ -67,7 +67,7 @@ export class VaaCache { this.ttl = ttl; } - set(key: VaaKey, publishTime: number, vaa: string) { + set(key: VaaKey, publishTime: number, vaa: string): void { if (this.cache[key]) { this.cache[key].push({ publishTime, vaa }); } else { @@ -75,16 +75,16 @@ export class VaaCache { } } - get(key: VaaKey, publishTime: number) { + get(key: VaaKey, publishTime: number): VaaConfig | undefined { if (!this.cache[key]) { - return null; + return undefined; } else { const vaaConf = this.find(this.cache[key], publishTime); return vaaConf; } } - find(arr: VaaConfig[], publishTime: number) { + find(arr: VaaConfig[], publishTime: number): VaaConfig | undefined { let left = 0; let right = arr.length - 1; let nextLargest = -1; @@ -101,7 +101,7 @@ export class VaaCache { } } - return nextLargest !== -1 ? arr[nextLargest] : null; + return nextLargest !== -1 ? arr[nextLargest] : undefined; } async removeExpiredValues() { @@ -325,7 +325,7 @@ export class Listener implements PriceStore { ); } - getVaa(priceFeedId: string, publishTime: number): VaaConfig | null { + getVaa(priceFeedId: string, publishTime: number): VaaConfig | undefined { return this.vaasCache.get(priceFeedId, publishTime); } From fc84fa86d06075091d495abae42cee59c18e96ab Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 07:57:30 +0000 Subject: [PATCH 10/21] use status code --- third_party/pyth/price-service/src/rest.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index 9ac2934ba5..719c7a9601 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -151,7 +151,7 @@ export class RestAPI { if (arr.length > 0 && arr[0]) { res.json(arr[0]); } else { - res.status(404).send("VAA not found"); + res.status(StatusCodes.NOT_FOUND).send("VAA not found"); } }); } else { From 7c1dd13b530127a49ad388d9712f0cd2107e6267 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 08:05:11 +0000 Subject: [PATCH 11/21] make web-api an env var --- .../price-service/src/__tests__/ws.test.ts | 2 +- third_party/pyth/price-service/src/rest.ts | 24 ++++++++++--------- tilt-devnet/k8s/pyth-price-service.yaml | 4 +++- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/third_party/pyth/price-service/src/__tests__/ws.test.ts b/third_party/pyth/price-service/src/__tests__/ws.test.ts index ce9eb63e2d..aad27090f9 100644 --- a/third_party/pyth/price-service/src/__tests__/ws.test.ts +++ b/third_party/pyth/price-service/src/__tests__/ws.test.ts @@ -108,7 +108,7 @@ beforeAll(async () => { getLatestPriceInfo: (_priceFeedId: string) => undefined, addUpdateListener: (_callback: (priceInfo: PriceInfo) => any) => undefined, getPriceIds: () => new Set(priceInfos.map((info) => info.priceFeed.id)), - getVaa: (_vaasCacheKey: string) => null, + getVaa: (_vaasCacheKey: string) => undefined, }; api = new WebSocketAPI(priceInfo); diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index 719c7a9601..bb607060fe 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -143,17 +143,19 @@ export class RestAPI { !vaa ) { // cache miss - fetch( - `https://web-api.pyth.network/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=pythnet` - ) - .then((r: any) => r.json()) - .then((arr: any) => { - if (arr.length > 0 && arr[0]) { - res.json(arr[0]); - } else { - res.status(StatusCodes.NOT_FOUND).send("VAA not found"); - } - }); + if (process.env["WEB_API_ENDPOINT"]) { + fetch( + `https://web-api.pyth.network/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=pythnet` + ) + .then((r: any) => r.json()) + .then((arr: any) => { + if (arr.length > 0 && arr[0]) { + res.json(arr[0]); + } else { + res.status(StatusCodes.NOT_FOUND).send("VAA not found"); + } + }); + } } else { // cache hit const processedVaa = { diff --git a/tilt-devnet/k8s/pyth-price-service.yaml b/tilt-devnet/k8s/pyth-price-service.yaml index 273e91f59a..1f82efa01c 100644 --- a/tilt-devnet/k8s/pyth-price-service.yaml +++ b/tilt-devnet/k8s/pyth-price-service.yaml @@ -79,4 +79,6 @@ spec: - name: PYTH_WRITER_INSERT_LATENCY_SECONDS value: "30" - name: CACHE_TTL_SECONDS - value: "300" \ No newline at end of file + value: "300" + - name: WEB_API_ENDPOINT + value: https://web-api.pyth.network \ No newline at end of file From 62295338f6373c96d6eb43c06f2398d365c60374 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 08:06:45 +0000 Subject: [PATCH 12/21] update env var --- third_party/pyth/price-service/docker-compose.mainnet.yaml | 1 + third_party/pyth/price-service/docker-compose.testnet.yaml | 3 ++- third_party/pyth/price-service/src/rest.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/third_party/pyth/price-service/docker-compose.mainnet.yaml b/third_party/pyth/price-service/docker-compose.mainnet.yaml index c8726e1ac6..e49f70fbbb 100644 --- a/third_party/pyth/price-service/docker-compose.mainnet.yaml +++ b/third_party/pyth/price-service/docker-compose.mainnet.yaml @@ -41,6 +41,7 @@ services: REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" PYTH_WRITER_INSERT_LATENCY_SECONDS: "30" CACHE_TTL_SECONDS: "300" + WEB_API_ENDPOINT: "https://web-api.pyth.network" healthcheck: test: [ diff --git a/third_party/pyth/price-service/docker-compose.testnet.yaml b/third_party/pyth/price-service/docker-compose.testnet.yaml index ab49c89d41..b91e3a42fe 100644 --- a/third_party/pyth/price-service/docker-compose.testnet.yaml +++ b/third_party/pyth/price-service/docker-compose.testnet.yaml @@ -37,10 +37,11 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning - CLUSTER: pythtest + CLUSTER: devnet REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" PYTH_WRITER_INSERT_LATENCY_SECONDS: "30" CACHE_TTL_SECONDS: "300" + WEB_API_ENDPOINT: "https://web-api.pyth.network" healthcheck: test: [ diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index bb607060fe..f42bd71722 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -145,7 +145,7 @@ export class RestAPI { // cache miss if (process.env["WEB_API_ENDPOINT"]) { fetch( - `https://web-api.pyth.network/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=pythnet` + `${process.env["WEB_API_ENDPOINT"]}/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=pythnet` ) .then((r: any) => r.json()) .then((arr: any) => { From 9379f8115d72c01cb092f9ad8a20dcae70cf7a6f Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 17:22:46 +0900 Subject: [PATCH 13/21] fix precommit --- tilt-devnet/k8s/pyth-price-service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tilt-devnet/k8s/pyth-price-service.yaml b/tilt-devnet/k8s/pyth-price-service.yaml index 1f82efa01c..c6b9e028de 100644 --- a/tilt-devnet/k8s/pyth-price-service.yaml +++ b/tilt-devnet/k8s/pyth-price-service.yaml @@ -81,4 +81,4 @@ spec: - name: CACHE_TTL_SECONDS value: "300" - name: WEB_API_ENDPOINT - value: https://web-api.pyth.network \ No newline at end of file + value: https://web-api.pyth.network From d1ca0bfe1ad978fb244d558ba9f11e2ea2fa33d5 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 19:40:14 +0900 Subject: [PATCH 14/21] fix precommit --- third_party/pyth/price-service/src/listen.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index b69f84397a..4f59a04399 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -170,7 +170,7 @@ export class Listener implements PriceStore { } async runCacheCleanupLoop(interval: number = 60) { - setInterval(this.vaasCache.removeExpiredValues, interval * 1000) + setInterval(this.vaasCache.removeExpiredValues, interval * 1000); } async run() { From a9ef4836aa81484220bcfb24bb9172d564496115 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:20:16 +0000 Subject: [PATCH 15/21] fix changes --- third_party/pyth/price-service/.env.sample | 2 +- .../price-service/docker-compose.mainnet.yaml | 5 +- .../price-service/docker-compose.testnet.yaml | 5 +- .../price-service/src/__tests__/rest.test.ts | 60 +++++++++---------- third_party/pyth/price-service/src/index.ts | 2 + third_party/pyth/price-service/src/listen.ts | 5 ++ third_party/pyth/price-service/src/rest.ts | 23 +++---- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/third_party/pyth/price-service/.env.sample b/third_party/pyth/price-service/.env.sample index 6a36c5d110..63d6072e13 100644 --- a/third_party/pyth/price-service/.env.sample +++ b/third_party/pyth/price-service/.env.sample @@ -18,7 +18,7 @@ PROM_PORT=8081 # The default is to log with level info. #LOG_LEVEL=debug -CLUSTER=pythnet +DB_API_CLUSTER=pythnet REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS=60 PYTH_WRITER_INSERT_LATENCY_SECONDS=30 CACHE_TTL_SECONDS=300 diff --git a/third_party/pyth/price-service/docker-compose.mainnet.yaml b/third_party/pyth/price-service/docker-compose.mainnet.yaml index e49f70fbbb..669d6693f0 100644 --- a/third_party/pyth/price-service/docker-compose.mainnet.yaml +++ b/third_party/pyth/price-service/docker-compose.mainnet.yaml @@ -37,11 +37,10 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning - CLUSTER: pythnet + DB_API_CLUSTER: pythnet REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" - PYTH_WRITER_INSERT_LATENCY_SECONDS: "30" CACHE_TTL_SECONDS: "300" - WEB_API_ENDPOINT: "https://web-api.pyth.network" + DB_API_ENDPOINT: "https://web-api.pyth.network" healthcheck: test: [ diff --git a/third_party/pyth/price-service/docker-compose.testnet.yaml b/third_party/pyth/price-service/docker-compose.testnet.yaml index b91e3a42fe..0fb189219c 100644 --- a/third_party/pyth/price-service/docker-compose.testnet.yaml +++ b/third_party/pyth/price-service/docker-compose.testnet.yaml @@ -37,11 +37,10 @@ services: READINESS_SPY_SYNC_TIME_SECONDS: "20" READINESS_NUM_LOADED_SYMBOLS: "50" LOG_LEVEL: warning - CLUSTER: devnet + DB_API_CLUSTER: devnet REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS: "60" - PYTH_WRITER_INSERT_LATENCY_SECONDS: "30" CACHE_TTL_SECONDS: "300" - WEB_API_ENDPOINT: "https://web-api.pyth.network" + DB_API_ENDPOINT: "https://web-api.pyth.network" healthcheck: test: [ diff --git a/third_party/pyth/price-service/src/__tests__/rest.test.ts b/third_party/pyth/price-service/src/__tests__/rest.test.ts index b03820136a..b067563bb1 100644 --- a/third_party/pyth/price-service/src/__tests__/rest.test.ts +++ b/third_party/pyth/price-service/src/__tests__/rest.test.ts @@ -125,37 +125,37 @@ describe("Latest Price Feed Endpoint", () => { }); }); -describe("Latest Vaa Bytes Endpoint", () => { - test("When called with valid ids, returns vaa bytes as array, merged if necessary", async () => { - const ids = [ - expandTo64Len("abcd"), - expandTo64Len("ef01"), - expandTo64Len("3456"), - ]; - const resp = await request(app).get("/api/latest_vaas").query({ ids }); - expect(resp.status).toBe(StatusCodes.OK); - expect(resp.body.length).toBe(2); - expect(resp.body).toContain( - Buffer.from("a1b2c3d4", "hex").toString("base64") - ); - expect(resp.body).toContain( - Buffer.from("bad01bad", "hex").toString("base64") - ); - }); +// describe("Latest Vaa Bytes Endpoint", () => { +// test("When called with valid ids, returns vaa bytes as array, merged if necessary", async () => { +// const ids = [ +// expandTo64Len("abcd"), +// expandTo64Len("ef01"), +// expandTo64Len("3456"), +// ]; +// const resp = await request(app).get("/api/latest_vaas").query({ ids }); +// expect(resp.status).toBe(StatusCodes.OK); +// expect(resp.body.length).toBe(2); +// expect(resp.body).toContain( +// Buffer.from("a1b2c3d4", "hex").toString("base64") +// ); +// expect(resp.body).toContain( +// Buffer.from("bad01bad", "hex").toString("base64") +// ); +// }); - test("When called with some non-existent ids within ids, returns error mentioning non-existent ids", async () => { - const ids = [ - expandTo64Len("ab01"), - expandTo64Len("3456"), - expandTo64Len("effe"), - ]; - const resp = await request(app).get("/api/latest_vaas").query({ ids }); - expect(resp.status).toBe(StatusCodes.BAD_REQUEST); - expect(resp.body.message).toContain(ids[0]); - expect(resp.body.message).not.toContain(ids[1]); - expect(resp.body.message).toContain(ids[2]); - }); -}); +// test("When called with some non-existent ids within ids, returns error mentioning non-existent ids", async () => { +// const ids = [ +// expandTo64Len("ab01"), +// expandTo64Len("3456"), +// expandTo64Len("effe"), +// ]; +// const resp = await request(app).get("/api/latest_vaas").query({ ids }); +// expect(resp.status).toBe(StatusCodes.BAD_REQUEST); +// expect(resp.body.message).toContain(ids[0]); +// expect(resp.body.message).not.toContain(ids[1]); +// expect(resp.body.message).toContain(ids[2]); +// }); +// }); // describe("Get Vaa Endpoint", () => { // test("When called with valid id and publish_time, returns vaa string if cached", async () => { diff --git a/third_party/pyth/price-service/src/index.ts b/third_party/pyth/price-service/src/index.ts index a05ed48a26..fcee332d42 100644 --- a/third_party/pyth/price-service/src/index.ts +++ b/third_party/pyth/price-service/src/index.ts @@ -52,6 +52,8 @@ async function run() { const restAPI = new RestAPI( { port: parseInt(envOrErr("REST_PORT"), 10), + dbApiEndpoint: process.env.DB_API_ENDPOINT, + dbApiCluster: process.env.DB_API_CLUSTER, }, listener, isReady, diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 4f59a04399..7f27a836a4 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -49,6 +49,8 @@ type ListenerConfig = { spyServiceHost: string; filtersRaw?: string; readiness: ListenerReadinessConfig; + webApiEndpoint?: string; + webApiCluster?: string; }; type VaaKey = string; @@ -85,6 +87,9 @@ export class VaaCache { } find(arr: VaaConfig[], publishTime: number): VaaConfig | undefined { + if (arr.length === 0 || publishTime < arr[0].publishTime) { + return undefined; + } let left = 0; let right = arr.length - 1; let nextLargest = -1; diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index f42bd71722..bd29b03295 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -13,10 +13,6 @@ import { logger } from "./logging"; import { PromClient } from "./promClient"; dotenv.config(); -const PYTH_WRITER_INSERT_LATENCY_SECONDS = Number( - envOrErr("PYTH_WRITER_INSERT_LATENCY_SECONDS") -); - const MORGAN_LOG_FORMAT = ':remote-addr - :remote-user ":method :url HTTP/:http-version"' + ' :status :res[content-length] :response-time ms ":referrer" ":user-agent"'; @@ -43,14 +39,18 @@ export class RestAPI { private priceFeedVaaInfo: PriceStore; private isReady: (() => boolean) | undefined; private promClient: PromClient | undefined; + private dbApiEndpoint?: string; + private dbApiCluster?: string; constructor( - config: { port: number }, + config: { port: number, dbApiEndpoint?: string, dbApiCluster?: string }, priceFeedVaaInfo: PriceStore, isReady?: () => boolean, - promClient?: PromClient + promClient?: PromClient, ) { this.port = config.port; + this.dbApiEndpoint = config.dbApiEndpoint; + this.dbApiCluster = config.dbApiCluster; this.priceFeedVaaInfo = priceFeedVaaInfo; this.isReady = isReady; this.promClient = promClient; @@ -136,16 +136,11 @@ export class RestAPI { const publishTime = Number(req.query.publish_time as string); const vaa = this.priceFeedVaaInfo.getVaa(priceFeedId, publishTime); // if publishTime is older than cache ttl or vaa is not found, fetch from db - if ( - publishTime < - Math.floor(Date.now() / 1000) - - PYTH_WRITER_INSERT_LATENCY_SECONDS || - !vaa - ) { + if (!vaa) { // cache miss - if (process.env["WEB_API_ENDPOINT"]) { + if (this.dbApiEndpoint && this.dbApiCluster) { fetch( - `${process.env["WEB_API_ENDPOINT"]}/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=pythnet` + `${this.dbApiEndpoint}/vaa?id=${priceFeedId}&publishTime=${publishTime}&cluster=${this.dbApiCluster}` ) .then((r: any) => r.json()) .then((arr: any) => { From 9076ee36b37c95502296357e7a0ce605a557fbad Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:25:20 +0000 Subject: [PATCH 16/21] remove env vars from tilt-devnet --- tilt-devnet/k8s/pyth-price-service.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tilt-devnet/k8s/pyth-price-service.yaml b/tilt-devnet/k8s/pyth-price-service.yaml index c6b9e028de..35cfdcac21 100644 --- a/tilt-devnet/k8s/pyth-price-service.yaml +++ b/tilt-devnet/k8s/pyth-price-service.yaml @@ -72,13 +72,7 @@ spec: value: "6" - name: LOG_LEVEL value: debug - - name: CLUSTER - value: devnet - name: REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS value: "60" - - name: PYTH_WRITER_INSERT_LATENCY_SECONDS - value: "30" - name: CACHE_TTL_SECONDS value: "300" - - name: WEB_API_ENDPOINT - value: https://web-api.pyth.network From 1f4abaea525539bdbeb3bc2a892fb692676e76a6 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:27:38 +0000 Subject: [PATCH 17/21] address changes --- third_party/pyth/price-service/.env.sample | 2 - .../price-service/src/__tests__/rest.test.ts | 74 ++++++++----------- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/third_party/pyth/price-service/.env.sample b/third_party/pyth/price-service/.env.sample index 63d6072e13..fed4e7e746 100644 --- a/third_party/pyth/price-service/.env.sample +++ b/third_party/pyth/price-service/.env.sample @@ -18,7 +18,5 @@ PROM_PORT=8081 # The default is to log with level info. #LOG_LEVEL=debug -DB_API_CLUSTER=pythnet REMOVE_EXPIRED_VALUES_INTERVAL_SECONDS=60 -PYTH_WRITER_INSERT_LATENCY_SECONDS=30 CACHE_TTL_SECONDS=300 diff --git a/third_party/pyth/price-service/src/__tests__/rest.test.ts b/third_party/pyth/price-service/src/__tests__/rest.test.ts index b067563bb1..b1ed473711 100644 --- a/third_party/pyth/price-service/src/__tests__/rest.test.ts +++ b/third_party/pyth/price-service/src/__tests__/rest.test.ts @@ -125,48 +125,34 @@ describe("Latest Price Feed Endpoint", () => { }); }); -// describe("Latest Vaa Bytes Endpoint", () => { -// test("When called with valid ids, returns vaa bytes as array, merged if necessary", async () => { -// const ids = [ -// expandTo64Len("abcd"), -// expandTo64Len("ef01"), -// expandTo64Len("3456"), -// ]; -// const resp = await request(app).get("/api/latest_vaas").query({ ids }); -// expect(resp.status).toBe(StatusCodes.OK); -// expect(resp.body.length).toBe(2); -// expect(resp.body).toContain( -// Buffer.from("a1b2c3d4", "hex").toString("base64") -// ); -// expect(resp.body).toContain( -// Buffer.from("bad01bad", "hex").toString("base64") -// ); -// }); - -// test("When called with some non-existent ids within ids, returns error mentioning non-existent ids", async () => { -// const ids = [ -// expandTo64Len("ab01"), -// expandTo64Len("3456"), -// expandTo64Len("effe"), -// ]; -// const resp = await request(app).get("/api/latest_vaas").query({ ids }); -// expect(resp.status).toBe(StatusCodes.BAD_REQUEST); -// expect(resp.body.message).toContain(ids[0]); -// expect(resp.body.message).not.toContain(ids[1]); -// expect(resp.body.message).toContain(ids[2]); -// }); -// }); +describe("Latest Vaa Bytes Endpoint", () => { + test("When called with valid ids, returns vaa bytes as array, merged if necessary", async () => { + const ids = [ + expandTo64Len("abcd"), + expandTo64Len("ef01"), + expandTo64Len("3456"), + ]; + const resp = await request(app).get("/api/latest_vaas").query({ ids }); + expect(resp.status).toBe(StatusCodes.OK); + expect(resp.body.length).toBe(2); + expect(resp.body).toContain( + Buffer.from("a1b2c3d4", "hex").toString("base64") + ); + expect(resp.body).toContain( + Buffer.from("bad01bad", "hex").toString("base64") + ); + }); -// describe("Get Vaa Endpoint", () => { -// test("When called with valid id and publish_time, returns vaa string if cached", async () => { -// const id = expandTo64Len("abcd"); -// const resp = await request(app) -// .get("/api/get_vaa") -// .query({ id, publish_time: 1 }); -// expect(resp.status).toBe(StatusCodes.OK); -// expect(resp.body.length).toBe(8); -// expect(resp.body).toContain( -// Buffer.from("a1b2c3d4", "hex").toString("base64") -// ); -// }); -// }); + test("When called with some non-existent ids within ids, returns error mentioning non-existent ids", async () => { + const ids = [ + expandTo64Len("ab01"), + expandTo64Len("3456"), + expandTo64Len("effe"), + ]; + const resp = await request(app).get("/api/latest_vaas").query({ ids }); + expect(resp.status).toBe(StatusCodes.BAD_REQUEST); + expect(resp.body.message).toContain(ids[0]); + expect(resp.body.message).not.toContain(ids[1]); + expect(resp.body.message).toContain(ids[2]); + }); +}); From 59859d53d7c819d7da32b336357e98db18c8f2cd Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:34:33 +0000 Subject: [PATCH 18/21] fix lint issues --- third_party/pyth/price-service/src/listen.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 7f27a836a4..230b0a14c8 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -61,27 +61,27 @@ type VaaConfig = { }; export class VaaCache { - private cache: { [key: string]: VaaConfig[] }; + private cache: Map; private ttl: number; constructor(ttl: number = 300) { - this.cache = {}; + this.cache = new Map(); this.ttl = ttl; } set(key: VaaKey, publishTime: number, vaa: string): void { - if (this.cache[key]) { - this.cache[key].push({ publishTime, vaa }); + if (this.cache.has(key)) { + this.cache.get(key)!.push({ publishTime, vaa }); } else { - this.cache[key] = [{ publishTime, vaa }]; + this.cache.set(key, [{ publishTime, vaa }]); } } get(key: VaaKey, publishTime: number): VaaConfig | undefined { - if (!this.cache[key]) { + if (!this.cache.has(key)) { return undefined; } else { - const vaaConf = this.find(this.cache[key], publishTime); + const vaaConf = this.find(this.cache.get(key)!, publishTime); return vaaConf; } } @@ -112,9 +112,9 @@ export class VaaCache { async removeExpiredValues() { const now = Math.floor(Date.now() / 1000); for (const key in this.cache) { - this.cache[key] = this.cache[key].filter( + this.cache.set(key, this.cache.get(key)!.filter( (vaaConf) => now - vaaConf.publishTime < this.ttl - ); + )); } } } From 4ddaa1e4460f1de1069a6071d7535de61d75f51c Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:36:24 +0000 Subject: [PATCH 19/21] fix linting issues again --- third_party/pyth/price-service/src/listen.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/third_party/pyth/price-service/src/listen.ts b/third_party/pyth/price-service/src/listen.ts index 230b0a14c8..ebdebe2cdd 100644 --- a/third_party/pyth/price-service/src/listen.ts +++ b/third_party/pyth/price-service/src/listen.ts @@ -111,10 +111,8 @@ export class VaaCache { async removeExpiredValues() { const now = Math.floor(Date.now() / 1000); - for (const key in this.cache) { - this.cache.set(key, this.cache.get(key)!.filter( - (vaaConf) => now - vaaConf.publishTime < this.ttl - )); + for (const arr of this.cache.values()) { + arr.filter((vaaConf) => now - vaaConf.publishTime < this.ttl); } } } From 55ab2ff652e6a32b1274146ff7ec3060fa119b00 Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:36:48 +0000 Subject: [PATCH 20/21] bump package version --- third_party/pyth/price-service/package-lock.json | 4 ++-- third_party/pyth/price-service/package.json | 2 +- third_party/pyth/price-service/src/rest.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/third_party/pyth/price-service/package-lock.json b/third_party/pyth/price-service/package-lock.json index 08561800ba..8b412016e7 100644 --- a/third_party/pyth/price-service/package-lock.json +++ b/third_party/pyth/price-service/package-lock.json @@ -1,12 +1,12 @@ { "name": "@pythnetwork/pyth-price-service", - "version": "2.2.4", + "version": "2.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@pythnetwork/pyth-price-service", - "version": "2.2.4", + "version": "2.3.0", "license": "Apache-2.0", "dependencies": { "@certusone/wormhole-sdk": "^0.1.4", diff --git a/third_party/pyth/price-service/package.json b/third_party/pyth/price-service/package.json index 02bbebf66b..a5fb3ed23d 100644 --- a/third_party/pyth/price-service/package.json +++ b/third_party/pyth/price-service/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/pyth-price-service", - "version": "2.2.4", + "version": "2.3.0", "description": "Pyth Price Service", "main": "index.js", "scripts": { diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index bd29b03295..a3de84cb88 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -43,10 +43,10 @@ export class RestAPI { private dbApiCluster?: string; constructor( - config: { port: number, dbApiEndpoint?: string, dbApiCluster?: string }, + config: { port: number; dbApiEndpoint?: string; dbApiCluster?: string }, priceFeedVaaInfo: PriceStore, isReady?: () => boolean, - promClient?: PromClient, + promClient?: PromClient ) { this.port = config.port; this.dbApiEndpoint = config.dbApiEndpoint; From 864157409e4c0b89311e9f575e71d1da5425ca4e Mon Sep 17 00:00:00 2001 From: Daniel Chew Date: Fri, 13 Jan 2023 14:38:33 +0000 Subject: [PATCH 21/21] remove unused imports --- third_party/pyth/price-service/src/rest.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/third_party/pyth/price-service/src/rest.ts b/third_party/pyth/price-service/src/rest.ts index a3de84cb88..a751cebf60 100644 --- a/third_party/pyth/price-service/src/rest.ts +++ b/third_party/pyth/price-service/src/rest.ts @@ -1,17 +1,15 @@ import { HexString } from "@pythnetwork/pyth-sdk-js"; import cors from "cors"; -import * as dotenv from "dotenv"; // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import express, { NextFunction, Request, Response } from "express"; import { Joi, schema, validate, ValidationError } from "express-validation"; import { Server } from "http"; import { StatusCodes } from "http-status-codes"; import morgan from "morgan"; import fetch from "node-fetch"; -import { envOrErr, TimestampInSec } from "./helpers"; +import { TimestampInSec } from "./helpers"; import { PriceStore } from "./listen"; import { logger } from "./logging"; import { PromClient } from "./promClient"; -dotenv.config(); const MORGAN_LOG_FORMAT = ':remote-addr - :remote-user ":method :url HTTP/:http-version"' +