diff --git a/Justfile b/Justfile index b8e7340de7..96526a0279 100644 --- a/Justfile +++ b/Justfile @@ -108,6 +108,7 @@ test-ts: test-e2e: yarn workspace @blocksense/e2e-tests run test:scenarios + yarn workspace @blocksense/e2e-tests run test:wit-scenarios [group('Working with oracles')] [doc('Build a specific oracle')] diff --git a/apps/data-feeds-config-generator/src/chainlink-compatibility/index.ts b/apps/data-feeds-config-generator/src/chainlink-compatibility/index.ts index 3d1f46df60..d7f58bebea 100644 --- a/apps/data-feeds-config-generator/src/chainlink-compatibility/index.ts +++ b/apps/data-feeds-config-generator/src/chainlink-compatibility/index.ts @@ -54,7 +54,7 @@ async function getBlocksenseFeedsCompatibility( } const dataFeedId = dataFeed.id; - const { base, quote } = dataFeed.additional_feed_info.pair; + const { base, quote } = dataFeed.additional_feed_info.pair!; const baseAddress = isSupportedCurrencySymbol(base) ? currencySymbolToDenominationAddress[base] diff --git a/apps/data-feeds-config-generator/src/generation/initial/data-providers.ts b/apps/data-feeds-config-generator/src/generation/initial/data-providers.ts index 6fd1bd42ed..79fbac3136 100644 --- a/apps/data-feeds-config-generator/src/generation/initial/data-providers.ts +++ b/apps/data-feeds-config-generator/src/generation/initial/data-providers.ts @@ -14,7 +14,7 @@ export async function addDataProviders( const dataFeedsWithCryptoResources = await Promise.all( filteredFeeds.map(async feed => { const providers = getAllProvidersForPair( - feed.additional_feed_info.pair, + feed.additional_feed_info.pair!, providersData, ); return { @@ -37,5 +37,5 @@ export async function addDataProviders( // Filter feeds that have a quote function filterFeedsWithQuotes(feeds: SimplifiedFeed[]): SimplifiedFeed[] { - return feeds.filter(feed => feed.additional_feed_info.pair.quote); + return feeds.filter(feed => feed.additional_feed_info.pair!.quote); } diff --git a/apps/data-feeds-config-generator/src/generation/initial/utils/common.ts b/apps/data-feeds-config-generator/src/generation/initial/utils/common.ts index eda58d8089..c71bd84541 100644 --- a/apps/data-feeds-config-generator/src/generation/initial/utils/common.ts +++ b/apps/data-feeds-config-generator/src/generation/initial/utils/common.ts @@ -12,7 +12,7 @@ export function getUniqueDataFeeds( const seenPairs = new Set(); return dataFeeds.filter(feed => { - const pairKey = pairToString(feed.additional_feed_info.pair); + const pairKey = pairToString(feed.additional_feed_info.pair!); if (seenPairs.has(pairKey)) { return false; @@ -27,7 +27,7 @@ export function addStableCoinVariants( feeds: SimplifiedFeed[], ): SimplifiedFeed[] { const stableCoinVariants = feeds.flatMap(feed => { - const { base, quote } = feed.additional_feed_info.pair; + const { base, quote } = feed.additional_feed_info.pair!; if (quote in stableCoins) { return stableCoins[quote as keyof typeof stableCoins] .map(altStableCoin => createPair(base, altStableCoin)) @@ -65,7 +65,7 @@ export async function addMarketCapRank( const asset = cmcMarketCap.find( asset => asset.symbol.toLowerCase() === - feed.additional_feed_info.pair.base.toLowerCase(), + feed.additional_feed_info.pair!.base.toLowerCase(), ); return { ...feed, diff --git a/apps/data-feeds-config-generator/src/generation/update/crypto-providers.ts b/apps/data-feeds-config-generator/src/generation/update/crypto-providers.ts index 26721c8626..9c43e789ca 100644 --- a/apps/data-feeds-config-generator/src/generation/update/crypto-providers.ts +++ b/apps/data-feeds-config-generator/src/generation/update/crypto-providers.ts @@ -30,7 +30,7 @@ export async function updateExchangesArgumentConfig(): Promise { for (const feed of feedsConfig.feeds) { const pair = feed.additional_feed_info.pair; const initialExchangePrices = getExchangesPriceDataForPair( - pair, + pair!, providersData, ); const outlierExchanges = detectPriceOutliers( @@ -44,14 +44,14 @@ export async function updateExchangesArgumentConfig(): Promise { normalizedProvidersData = removePriceOutliers( providersData, - pair, + pair!, outlierExchanges, ); } const updatedFeedConfig = feedsConfig.feeds.map(feed => { const providers = getAllProvidersForPair( - feed.additional_feed_info.pair, + feed.additional_feed_info.pair!, normalizedProvidersData, ); diff --git a/apps/dev/src/commands/decoder/generate-decoder/generate-decoder.ts b/apps/dev/src/commands/decoder/generate-decoder/generate-decoder.ts index 89977992c9..3a3bb55a4c 100644 --- a/apps/dev/src/commands/decoder/generate-decoder/generate-decoder.ts +++ b/apps/dev/src/commands/decoder/generate-decoder/generate-decoder.ts @@ -1,21 +1,19 @@ -import { exec } from 'child_process'; -import fs from 'fs/promises'; import path from 'path'; -import { promisify } from 'util'; import { Effect } from 'effect'; import { Command, Options } from '@effect/cli'; +import { Command as Exec } from '@effect/platform'; +import * as FileSystem from '@effect/platform/FileSystem'; import chalk from 'chalk'; import { rootDir, valuesOf } from '@blocksense/base-utils'; import { expandJsonFields } from '@blocksense/decoders/expand-wit-json'; import { generateDecoders } from '@blocksense/decoders/generate-decoders'; -const execPromise = promisify(exec); - export const generateDecoder = Command.make( 'generate-decoder', { + stride: Options.integer('stride').pipe(Options.withDefault(0)), decoderType: Options.choice('decoder-type', ['ssz', 'encode-packed']).pipe( Options.withDefault('ssz'), ), @@ -35,28 +33,51 @@ export const generateDecoder = Command.make( Options.withDefault('generated-decoders'), ), }, - ({ decoderType, evmVersion, outputDir, witFunction, witPath, witWorld }) => + ({ + decoderType, + evmVersion, + outputDir, + stride, + witFunction, + witPath, + witWorld, + }) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; const witFilePath = path.join(rootDir, witPath); - if (!(yield* Effect.tryPromise(() => fs.stat(witFilePath)))) { + if (!(yield* fs.stat(witFilePath))) { return yield* Effect.fail(`WIT file not found at path: ${witFilePath}`); } + // Generate temporary JSON representation from WIT file + const tempOutputPath = path.join(rootDir, 'tmp/wit-output.json'); + + yield* fs.makeDirectory(path.dirname(tempOutputPath), { + recursive: true, + }); + const args = [ '--input', path.join(rootDir, witPath), + '--output', + tempOutputPath, '--world', witWorld, '--function', witFunction, ]; - const res = yield* Effect.tryPromise(() => - execPromise(`wit-converter ${args.join(' ')}`, { - cwd: path.join(rootDir, 'apps/wit-converter'), - }), + + yield* Exec.make('wit-converter', ...args).pipe( + Exec.workingDirectory(path.join(rootDir, 'apps/wit-converter')), + Exec.string, + ); + + const witJson = yield* fs.readFileString(tempOutputPath).pipe( + Effect.map(json => JSON.parse(json)), + // Clean up temporary file + Effect.ensuring(fs.remove(tempOutputPath).pipe(Effect.ignore)), ); - const witJson = JSON.parse(res.stdout); const containsUnion = valuesOf(witJson.types).some( (field: any) => field.type === 'union', ); @@ -69,7 +90,7 @@ export const generateDecoder = Command.make( const fields = expandJsonFields(witJson.payloadTypeName, witJson.types); const outputPath = path.join(rootDir, outputDir); - yield* Effect.tryPromise(() => fs.mkdir(outputPath, { recursive: true })); + yield* fs.makeDirectory(outputPath, { recursive: true }); const contractPaths = yield* Effect.tryPromise(() => generateDecoders( @@ -77,7 +98,7 @@ export const generateDecoder = Command.make( evmVersion, outputPath, fields[witJson.payloadTypeName], - { containsUnion }, + { containsUnion, stride }, ), ); diff --git a/apps/docs.blocksense.network/components/DataFeeds/Cards/PriceFeedConfigCard.tsx b/apps/docs.blocksense.network/components/DataFeeds/Cards/PriceFeedConfigCard.tsx index 9e5d8ef48e..bd73067c95 100644 --- a/apps/docs.blocksense.network/components/DataFeeds/Cards/PriceFeedConfigCard.tsx +++ b/apps/docs.blocksense.network/components/DataFeeds/Cards/PriceFeedConfigCard.tsx @@ -13,8 +13,8 @@ export const PriceFeedConfigCard = ({ feed }: DataFeedCardProps) => { title: 'Price Feed Configuration', description: '', items: [ - { label: 'Base', value: feed.additional_feed_info.pair.base }, - { label: 'Quote', value: feed.additional_feed_info.pair.quote }, + { label: 'Base', value: feed.additional_feed_info.pair?.base }, + { label: 'Quote', value: feed.additional_feed_info.pair?.quote }, { label: 'Decimals', value: feed.additional_feed_info.decimals }, { label: 'Category', value: feed.additional_feed_info.category }, ], diff --git a/apps/e2e-tests/.gitignore b/apps/e2e-tests/.gitignore new file mode 100644 index 0000000000..da9f8cc52e --- /dev/null +++ b/apps/e2e-tests/.gitignore @@ -0,0 +1,2 @@ +generated-decoders/ +cache/ diff --git a/apps/e2e-tests/package.json b/apps/e2e-tests/package.json index ecf087e0c8..74df83bdf2 100644 --- a/apps/e2e-tests/package.json +++ b/apps/e2e-tests/package.json @@ -6,12 +6,14 @@ "ts": "yarn node --import tsx", "test": "vitest run --typecheck --coverage", "test:unit": "yarn vitest -w false src/utils", - "test:scenarios": "yarn test src/test-scenarios/**/e2e.spec.ts" + "test:scenarios": "yarn test src/test-scenarios/general/e2e.spec.ts", + "test:wit-scenarios": "yarn test src/test-scenarios/wit/e2e.spec.ts" }, "dependencies": { "@blocksense/base-utils": "workspace:*", "@blocksense/config-types": "workspace:*", "@blocksense/contracts": "workspace:*", + "@blocksense/dev": "workspace:*", "@chainsafe/bls": "^8.2.0", "@effect/cluster": "^0.48.2", "@effect/experimental": "^0.54.6", @@ -26,6 +28,7 @@ "parse-prometheus-text-format": "^1.1.1", "tsx": "^4.20.5", "typescript": "5.9.2", + "viem": "^2.38.6", "vitest": "^3.2.4" }, "devDependencies": { diff --git a/apps/e2e-tests/src/test-scenarios/general/e2e.spec.ts b/apps/e2e-tests/src/test-scenarios/general/e2e.spec.ts index fa7b36b420..66cb0c4bc6 100644 --- a/apps/e2e-tests/src/test-scenarios/general/e2e.spec.ts +++ b/apps/e2e-tests/src/test-scenarios/general/e2e.spec.ts @@ -40,7 +40,8 @@ import { rgSearchPattern } from '../../utils/utilities'; import { expectedPCStatuses03 } from './expected-service-status'; describe.sequential('E2E Tests with process-compose', () => { - const testEnvironment = `e2e-general`; + const testScenario = 'general'; + const testEnvironment = `e2e-${testScenario}`; const network = 'ink_sepolia'; const MAX_HISTORY_ELEMENTS_PER_FEED = 8192; @@ -67,7 +68,7 @@ describe.sequential('E2E Tests with process-compose', () => { const res = await pipe( Effect.gen(function* () { processCompose = yield* EnvironmentManager; - yield* processCompose.start(testEnvironment); + yield* processCompose.start(testScenario); hasProcessComposeStarted = true; if (!process.listenerCount('SIGINT')) { @@ -253,7 +254,8 @@ describe.sequential('E2E Tests with process-compose', () => { // Make sure that the feeds info is updated for (const [id, data] of entriesOf(feedInfoAfterUpdates)) { - const { round: roundAfterUpdates, value: valueAfterUpdates } = data; + const { round: roundAfterUpdates, value } = data; + const valueAfterUpdates = Number(value.slice(0, 50)); // Pegged asset with 10% tolerance should be pegged // Pegged asset with 0.000001% tolerance should not be pegged if (id === '50000') { @@ -268,11 +270,22 @@ describe.sequential('E2E Tests with process-compose', () => { feed => feed.update_number === updatesToNetworks[network]['0:' + id] - 1, ); + const decimals = feedsConfig.feeds.find(f => f.id.toString() === id)! .additional_feed_info.decimals; - expect(valueAfterUpdates / 10 ** decimals).toBeCloseTo( - historyData!.value.Numerical, + if ( + typeof historyData!.value !== 'object' || + !('Numerical' in historyData!.value) + ) { + return yield* Effect.fail( + new Error('Unexpected feed value type in history'), + ); + } + + const historyValue = historyData!.value.Numerical; + expect(Number(valueAfterUpdates) / 10 ** decimals).toBeCloseTo( + historyValue, ); const expectedNumberOfUpdates = diff --git a/apps/e2e-tests/src/test-scenarios/wit/e2e.spec.ts b/apps/e2e-tests/src/test-scenarios/wit/e2e.spec.ts new file mode 100644 index 0000000000..4a28575c3b --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/e2e.spec.ts @@ -0,0 +1,418 @@ +import { deepStrictEqual } from 'assert'; +import { join } from 'path'; + +import { Effect, Exit, Layer, pipe, Schedule } from 'effect'; +import { Command, FileSystem } from '@effect/platform'; +import { NodeContext } from '@effect/platform-node'; +import { afterAll, beforeAll, describe, expect, it } from '@effect/vitest'; + +import { rootDir } from '@blocksense/base-utils'; +import { + entriesOf, + fromEntries, + valuesOf, +} from '@blocksense/base-utils/array-iter'; +import { + type EthereumAddress, + parseEthereumAddress, +} from '@blocksense/base-utils/evm'; +import type { NewFeedsConfig } from '@blocksense/config-types/data-feeds-config'; +import type { SequencerConfigV2 } from '@blocksense/config-types/node-config'; +import { createViemClient } from '@blocksense/contracts/viem'; + +import { + parseProcessesStatus, + ProcessComposeLive, +} from '../../utils/environment-managers/process-compose-manager'; +import type { EnvironmentManagerService } from '../../utils/environment-managers/types'; +import { EnvironmentManager } from '../../utils/environment-managers/types'; +import { + createGatewayController, + gateEffect, + installGateway, +} from '../../utils/services/gateway'; +import type { FeedsValueAndRound } from '../../utils/services/onchain'; +import { getDataFeedsInfoFromNetwork } from '../../utils/services/onchain'; +import type { SequencerService } from '../../utils/services/sequencer'; +import { Sequencer } from '../../utils/services/sequencer'; +import { expectedPCStatuses03 } from '../general/expected-service-status'; + +describe.sequential('E2E Tests with process-compose', () => { + const testScenario = `wit`; + const testEnvironment = `e2e-${testScenario}`; + const network = 'ink_sepolia'; + + const failFastGateway = createGatewayController(); + installGateway( + failFastGateway, + 'Skipping remaining tests because the gate test failed', + ); + + let sequencer: SequencerService; + let processCompose: EnvironmentManagerService; + let hasProcessComposeStarted = false; + + let sequencerConfig: SequencerConfigV2; + let feedsConfig: NewFeedsConfig; + + let feedIds: bigint[]; + let contractAddress: EthereumAddress; + + let initialFeedsInfo: FeedsValueAndRound; + + let existingFiles: string[] = []; + + const dirPath = join(rootDir, '/apps/e2e-tests/src/test-scenarios/wit/'); + + const deleteTestFiles = () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const allFiles = yield* fs.readDirectory(dirPath); + const newFiles = allFiles.filter(file => !existingFiles.includes(file)); + if (newFiles.length > 0) { + for (const file of newFiles) { + yield* fs.remove(join(dirPath, file), { + force: true, + recursive: true, + }); + console.log( + `Cleaning up ${newFiles.length} files created during tests:`, + `${newFiles.map(f => `\n - ${f}`).join('')}`, + ); + } + } + }); + + beforeAll(async () => { + // track files created during the tests + existingFiles = await Effect.runPromise( + FileSystem.FileSystem.pipe( + Effect.flatMap(fs => fs.readDirectory(dirPath)), + Effect.provide(NodeContext.layer), + ), + ); + + const res = await pipe( + Effect.gen(function* () { + processCompose = yield* EnvironmentManager; + yield* processCompose.start(testScenario); + hasProcessComposeStarted = true; + + if (!process.listenerCount('SIGINT')) { + process.once('SIGINT', () => { + if (hasProcessComposeStarted) { + Effect.runPromise( + processCompose + .stop() + .pipe(Effect.catchAll(() => Effect.succeed(undefined))) + .pipe(() => + deleteTestFiles().pipe(Effect.provide(NodeContext.layer)), + ), + ).finally(async () => { + process.exit(130); + }); + } else { + process.exit(130); + } + }); + } + + sequencer = yield* Sequencer; + }), + Effect.provide(Layer.merge(ProcessComposeLive, Sequencer.Live)), + Effect.runPromiseExit, + ); + + if (Exit.isFailure(res)) { + throw new Error(`Failed to start test environment: ${testEnvironment}`); + } + }); + + afterAll(() => + Effect.gen(function* () { + if (hasProcessComposeStarted) { + yield* processCompose.stop(); + } + + yield* deleteTestFiles(); + }) + .pipe(Effect.provide(NodeContext.layer)) + .pipe(Effect.runPromise), + ); + + it.live('Test processes state shortly after start', () => + gateEffect( + failFastGateway, + Effect.gen(function* () { + const equal = yield* Effect.retry( + processCompose + .getProcessesStatus() + .pipe( + Effect.tap(processes => + Effect.try(() => + deepStrictEqual(processes, expectedPCStatuses03), + ), + ), + ), + { + schedule: Schedule.fixed(1000), + times: 90, + }, + ); + // still validate the result + expect(equal).toBeTruthy(); + }).pipe(Effect.provide(ProcessComposeLive)), + 'Gate test failed: processes are not in expected state', + ), + ); + + it.live('Test sequencer configs are available and in correct format', () => + Effect.gen(function* () { + sequencerConfig = yield* sequencer.getConfig(); + feedsConfig = yield* sequencer.getFeedsConfig(); + expect(sequencerConfig).toBeTypeOf('object'); + expect(feedsConfig).toBeTypeOf('object'); + + contractAddress = parseEthereumAddress( + sequencerConfig.providers[network].contracts.find( + c => c.name === 'AggregatedDataFeedStore', + )?.address, + ); + + const allow_feeds = sequencerConfig.providers[network].allow_feeds; + feedIds = allow_feeds?.length + ? (allow_feeds as bigint[]) + : feedsConfig.feeds.map(feed => { + const stride = BigInt(feed.stride) << 120n; + return stride | feed.id; + }); + }).pipe( + Effect.tap( + Effect.gen(function* () { + const url = sequencerConfig.providers[network].url; + + // Fetch the initial round data for the feeds from the local network ( anvil ) + initialFeedsInfo = yield* getDataFeedsInfoFromNetwork( + feedIds, + contractAddress, + url, + ); + + // Enable the provider which is disabled by default ( ink_sepolia ) + yield* sequencer.enableProvider(network); + }), + ), + ), + ); + + it.live('Test sports db yields metrics', () => + Effect.gen(function* () { + yield* Effect.retry( + sequencer.fetchUpdatesToNetworksMetric().pipe( + Effect.filterOrFail(updates => { + // TODO: how to look for stride too? + return valuesOf(updates[network]).every(v => v >= 1); + }), + ), + { + schedule: Schedule.fixed(10000), + times: 30, + }, + ); + + const processes = yield* parseProcessesStatus(); + + expect(processes).toEqual(expectedPCStatuses03); + }), + ); + + it.live('Test feeds data is updated on the local network', () => + Effect.gen(function* () { + const url = sequencerConfig.providers[network].url; + + // Save map of initial rounds for each feed + const initialRounds = fromEntries( + entriesOf(initialFeedsInfo).map(([id, data]) => [id, data.round]), + ); + + // Get feeds information from the local network ( anvil ) + // for the same round as the initial one, to confirm it is not being overwritten + const initialFeedsInfoLocal = yield* getDataFeedsInfoFromNetwork( + feedIds, + contractAddress, + url, + initialRounds, + ); + + const latestFeedsInfoLocal = yield* getDataFeedsInfoFromNetwork( + feedIds, + contractAddress, + url, + ); + + expect(initialFeedsInfo).toEqual(initialFeedsInfoLocal); + expect(initialFeedsInfoLocal[feedIds[0].toString()].round).toEqual( + latestFeedsInfoLocal[feedIds[0].toString()].round - 1, + ); + expect(feedIds.length).toEqual(1); + + for (const feedId of feedIds) { + const value = latestFeedsInfoLocal[feedId.toString()].value; + const stride = feedId >> 120n; + const joinedValue = Array.isArray(value) + ? `0x${value.map(val => val.slice(2)).join('')}` + : value; + + expect(BigInt((joinedValue.length - 2) / 2 / 32)).toEqual(2n ** stride); + + yield* Command.make( + 'just', + 'dev', + 'decoder', + 'generate-decoder', + '--wit-path', + 'apps/e2e-tests/src/test-scenarios/wit/sports.wit', + '--output-dir', + 'apps/e2e-tests/src/test-scenarios/wit/generated-decoders', + '--stride', + stride.toString(), + ).pipe(Command.string()); + + yield* Command.make( + 'forge', + 'build', + '--root', + rootDir + '/apps/e2e-tests/src/test-scenarios/wit', + 'generated-decoders', + ).pipe(Command.string); + + const contracts = yield* FileSystem.FileSystem.pipe( + Effect.flatMap(fs => + fs + .readDirectory(dirPath + 'generated-decoders') + .pipe(Effect.map(files => files.map(file => file))), + ), + ); + + expect(contracts.length).toBeGreaterThan(0); + + for (const contractFile of contracts) { + const deployResult = yield* Command.make( + 'forge', + 'create', + '--rpc-url', + 'http://localhost:8500', + // sequencerConfig.providers[network].url, + // 1st account private key from anvil + '--private-key', + '0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', + '--root', + rootDir + '/apps/e2e-tests/src/test-scenarios/wit', + `generated-decoders/${contractFile}:${contractFile.replace('.sol', '')}`, + '--broadcast', + ).pipe(Command.string()); + + // Extract contract address from the deploy result + const contractAddressMatch = deployResult.match( + /Deployed to:\s*(0x[a-fA-F0-9]{40})/, + ); + if (!contractAddressMatch) { + throw new Error( + 'Failed to extract contract address from deploy result', + ); + } + + const contractAddress = contractAddressMatch[1]; + expect(contractAddress).toMatch(/^0x[a-fA-F0-9]{40}$/); + + const codeResult = yield* Command.make( + 'cast', + 'code', + '--rpc-url', + 'http://localhost:8500', + contractAddress, + ).pipe(Command.string); + + expect(codeResult).not.toEqual('0x'); + expect(codeResult.length).toBeGreaterThan(10); + + const abi = yield* Command.make( + 'forge', + 'inspect', + '--root', + rootDir + '/apps/e2e-tests/src/test-scenarios/wit', + 'generated-decoders/SSZDecoder.sol', + 'abi', + '--json', + ) + .pipe(Command.string) + .pipe(Effect.map(JSON.parse)); + + const viemClient = createViemClient(new URL('http://localhost:8500')); + + const decoded = (yield* Effect.tryPromise(() => + viemClient.readContract({ + address: contractAddress as EthereumAddress, + abi, + functionName: 'decode', + args: [joinedValue], + }), + )) as { + eventName: string; + season: string; + homeTeam: string; + awayTeam: string; + homeScore: bigint; + awayScore: bigint; + }; + + const eventId = yield* Effect.tryPromise({ + try: () => + fetch( + 'https://www.thesportsdb.com/api/v1/json/123/eventslast.php?id=133602', + { + method: 'GET', + }, + ).then(res => + res.json().then(data => data.results[0].idEvent as string), + ), + catch: () => { + throw new Error('Failed to fetch data from TheSportsDB API'); + }, + }); + + const event = yield* Effect.tryPromise({ + try: () => + fetch( + `https://www.thesportsdb.com/api/v1/json/123/lookupevent.php?id=${eventId}`, + { + method: 'GET', + }, + ).then(res => + res.json().then(data => { + return { + name: data.events[0].strEvent, + season: data.events[0].strSeason, + homeTeam: data.events[0].strHomeTeam, + awayTeam: data.events[0].strAwayTeam, + homeScore: data.events[0].intHomeScore, + awayScore: data.events[0].intAwayScore, + }; + }), + ), + catch: () => { + throw new Error('Failed to fetch data from TheSportsDB API'); + }, + }); + + expect(decoded.eventName).toEqual(event.name); + expect(decoded.season).toEqual(event.season); + expect(decoded.homeTeam).toEqual(event.homeTeam); + expect(decoded.awayTeam).toEqual(event.awayTeam); + expect(BigInt(decoded.homeScore)).toEqual(BigInt(event.homeScore)); + expect(BigInt(decoded.awayScore)).toEqual(BigInt(event.awayScore)); + } + } + }).pipe(Effect.provide(NodeContext.layer)), + ); +}); diff --git a/apps/e2e-tests/src/test-scenarios/wit/environment-setup.nix b/apps/e2e-tests/src/test-scenarios/wit/environment-setup.nix new file mode 100644 index 0000000000..8c52395517 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/environment-setup.nix @@ -0,0 +1,62 @@ +{ + config, + lib, + ... +}: +let + readPortsFromFile = + path: + let + content = builtins.readFile path; + lines = lib.strings.splitString "\n" content; + nonEmpty = builtins.filter (s: s != "") lines; + asInts = builtins.map builtins.fromJSON nonEmpty; + in + asInts; + + root = ../../../../..; + + availablePorts = + let + filePath = "${config.devenv.root}/config/generated/process-compose/e2e-wit/available-ports"; + ports = if builtins.pathExists filePath then readPortsFromFile filePath else [ ]; + in + if builtins.length ports > 0 then ports else [ 8547 ]; + anvilInkSepoliaPort = builtins.elemAt availablePorts 0; +in +{ + imports = [ + ../general/environment-setup.nix + ]; + + services.blocksense = { + logsDir = lib.mkForce "$GIT_ROOT/logs/process-compose/e2e-wit"; + + anvil.ink-sepolia = { + port = anvilInkSepoliaPort; + state = lib.mkForce "${config.devenv.root}/config/generated/process-compose/e2e-wit/anvil/state.json"; + }; + + sequencer.providers.ink-sepolia = { + publishing-criteria = lib.mkForce [ + { + feed-id = 0; # Sports DB + stride = 4; + peg-to-value = 1.00; + peg-tolerance-percentage = 10.0; # 10% tolerance assures that the price will be pegged + } + ]; + }; + + reporters.a.api-keys = lib.mkForce { }; + + oracles = lib.mkForce { + sports-db = { + exec-interval = 120; + allowed-outbound-hosts = [ + "https://www.thesportsdb.com" + ]; + }; + }; + }; +} diff --git a/apps/e2e-tests/src/test-scenarios/wit/feeds_config_v2.json b/apps/e2e-tests/src/test-scenarios/wit/feeds_config_v2.json new file mode 100644 index 0000000000..d04fc79d88 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/feeds_config_v2.json @@ -0,0 +1,32 @@ +{ + "feeds": [ + { + "id": "0", + "full_name": "Liverpool Matches Results", + "description": "Results of Liverpool matches", + "type": "sport-feed", + "oracle_id": "sports-db", + "value_type": "bytes", + "stride": 4, + "quorum": { + "percentage": 90, + "aggregation": "majority" + }, + "schedule": { + "interval_ms": 10000, + "heartbeat_ms": 30000, + "deviation_percentage": 0.1, + "first_report_start_unix_time_ms": 0 + }, + "additional_feed_info": { + "decimals": 0, + "category": "Sports", + "arguments": { + "kind": "sports-db", + "team_id": 133602, + "sport_type": "Soccer" + } + } + } + ] +} diff --git a/apps/e2e-tests/src/test-scenarios/wit/sports.wit b/apps/e2e-tests/src/test-scenarios/wit/sports.wit new file mode 100644 index 0000000000..725456efe0 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/sports.wit @@ -0,0 +1,17 @@ +package blocksense:oracle@2.0.0; + +interface oracle-types { + record payload { + event-name: string, + season: string, + home-team: string, + away-team: string, + home-score: u64, + away-score: u64, + } +} + +world blocksense-oracle { + use oracle-types.{payload}; + export handle-oracle-request: func() -> result; +} diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/ALPHAVANTAGE_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/ALPHAVANTAGE_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/ALPHAVANTAGE_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/APCA_API_KEY_ID b/apps/e2e-tests/src/test-scenarios/wit/test-keys/APCA_API_KEY_ID new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/APCA_API_KEY_ID @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/APCA_API_SECRET_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/APCA_API_SECRET_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/APCA_API_SECRET_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/CMC_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/CMC_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/CMC_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/FMP_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/FMP_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/FMP_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/METALS_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/METALS_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/METALS_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/SPOUT_RWA_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/SPOUT_RWA_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/SPOUT_RWA_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/TWELVEDATA_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/TWELVEDATA_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/TWELVEDATA_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/test-scenarios/wit/test-keys/YAHOO_FINANCE_API_KEY b/apps/e2e-tests/src/test-scenarios/wit/test-keys/YAHOO_FINANCE_API_KEY new file mode 100644 index 0000000000..587be6b4c3 --- /dev/null +++ b/apps/e2e-tests/src/test-scenarios/wit/test-keys/YAHOO_FINANCE_API_KEY @@ -0,0 +1 @@ +x diff --git a/apps/e2e-tests/src/utils/environment-managers/process-compose-manager.ts b/apps/e2e-tests/src/utils/environment-managers/process-compose-manager.ts index 1c026c5594..212cbf70a7 100644 --- a/apps/e2e-tests/src/utils/environment-managers/process-compose-manager.ts +++ b/apps/e2e-tests/src/utils/environment-managers/process-compose-manager.ts @@ -7,7 +7,7 @@ import { Stream, String as EString, } from 'effect'; -import { Command } from '@effect/platform'; +import { Command, FileSystem } from '@effect/platform'; import { NodeContext } from '@effect/platform-node'; import { arrayToObject } from '@blocksense/base-utils/array-iter'; @@ -18,6 +18,17 @@ import { logTestEnvironmentInfo } from '../utilities'; import { EnvironmentError, EnvironmentManager } from './types'; export const GENERAL_SCENARIO_FEEDS_CONFIG_DIR = `${rootDir}/apps/e2e-tests/src/test-scenarios/general`; +export function getFeedsConfigForScenario(scenario: string) { + return Effect.gen(function* () { + const configDir = `${rootDir}/apps/e2e-tests/src/test-scenarios/${scenario}`; + const exists = yield* FileSystem.FileSystem.pipe( + Effect.flatMap(fs => fs.exists(configDir)), + Effect.provide(NodeContext.layer), + ); + + return exists ? configDir : GENERAL_SCENARIO_FEEDS_CONFIG_DIR; + }); +} export const ProcessComposeLive = Layer.succeed( EnvironmentManager, @@ -56,7 +67,7 @@ export const ProcessComposeLive = Layer.succeed( }), ); -function startEnvironment(testEnvironment: string): Effect.Effect { +function startEnvironment(testScenario: string): Effect.Effect { const runString = ( stream: Stream.Stream, ): Effect.Effect => @@ -65,16 +76,20 @@ function startEnvironment(testEnvironment: string): Effect.Effect { Stream.runFold(EString.empty, EString.concat), ); + // NOTE: as per <../../../../../nix/test-environments/default.nix:39> + const testEnvironment = `e2e-${testScenario}`; + return Effect.gen(function* () { yield* logTestEnvironmentInfo('Starting', testEnvironment); - yield* Effect.sync(() => { - process.env['FEEDS_CONFIG_DIR'] = GENERAL_SCENARIO_FEEDS_CONFIG_DIR; - }); + + process.env['FEEDS_CONFIG_DIR'] = + yield* getFeedsConfigForScenario(testScenario); + const program = Effect.gen(function* () { const command = Command.make( 'just', 'start-environment', - 'e2e-general', + testEnvironment, '0', '--detached', ); @@ -122,7 +137,7 @@ export const ProcessComposeStatusSchema = S.mutable( S.Struct({ name: S.String, // namespace: S.String, - status: S.Literal('Running', 'Completed', 'Pending'), + status: S.Literal('Running', 'Completed', 'Skipped', 'Pending'), // system_time: S.String, // age: S.Number, // is_ready: S.String, diff --git a/apps/e2e-tests/src/utils/services/onchain.spec.ts b/apps/e2e-tests/src/utils/services/onchain.spec.ts index e3d1c53e55..4fbe27fa56 100644 --- a/apps/e2e-tests/src/utils/services/onchain.spec.ts +++ b/apps/e2e-tests/src/utils/services/onchain.spec.ts @@ -21,7 +21,6 @@ describe('getDataFeedsInfoFromNetwork', () => { const data = '0x0000000000000000000000000000000000000a3549cbad30000001986c265bf0'; - const value = 11223987629360; beforeEach(() => { vi.spyOn(AggregatedDataFeedStoreConsumer, 'create').mockReturnValue( @@ -49,7 +48,7 @@ describe('getDataFeedsInfoFromNetwork', () => { ); const expected: FeedsValueAndRound = { - '1': { value, round: 42 }, + '1': { value: data, round: 42 }, }; expect(result).toEqual(expected); @@ -74,7 +73,7 @@ describe('getDataFeedsInfoFromNetwork', () => { ); const expected: FeedsValueAndRound = { - '2': { value, round: 99 }, + '2': { value: data, round: 99 }, }; expect(result).toEqual(expected); diff --git a/apps/e2e-tests/src/utils/services/onchain.ts b/apps/e2e-tests/src/utils/services/onchain.ts index 376eb6856f..09ae5113e7 100644 --- a/apps/e2e-tests/src/utils/services/onchain.ts +++ b/apps/e2e-tests/src/utils/services/onchain.ts @@ -1,11 +1,12 @@ import { Effect } from 'effect'; +import type { HexDataString } from '@blocksense/base-utils'; import type { EthereumAddress, NetworkName } from '@blocksense/base-utils/evm'; import { AggregatedDataFeedStoreConsumer } from '@blocksense/contracts/viem'; export type FeedsValueAndRound = Record< string, - { value: number; round: number } + { value: HexDataString | HexDataString[]; round: number } >; /** @@ -33,26 +34,36 @@ export function getDataFeedsInfoFromNetwork( const feedsInfo: FeedsValueAndRound = {}; for (const feedId of feedIds) { - const data = yield* Effect.tryPromise(() => + const data = yield* Effect.tryPromise(() => { // If round is provided, fetch data at that specific round // Otherwise, fetch the latest data and index - roundsInfo !== undefined - ? ADFSConsumer.getSingleDataAtIndex( - feedId, - roundsInfo[feedId.toString()], - ).then(res => ({ - data: res, - index: roundsInfo[feedId.toString()], - })) - : ADFSConsumer.getLatestSingleDataAndIndex(BigInt(feedId)), - ).pipe( + if (roundsInfo !== undefined) { + return ( + feedId < 1n << 120n + ? ADFSConsumer.getSingleDataAtIndex( + feedId, + roundsInfo[feedId.toString()], + ) + : ADFSConsumer.getDataAtIndex( + feedId, + roundsInfo[feedId.toString()], + ) + ).then(res => ({ + data: res, + index: roundsInfo[feedId.toString()], + })); + } + return feedId < 1n << 120n + ? ADFSConsumer.getLatestSingleDataAndIndex(BigInt(feedId)) + : ADFSConsumer.getLatestDataAndIndex(BigInt(feedId)); + }).pipe( Effect.mapError( error => new Error(`Failed to fetch data for feed ${feedId}: ${error}`), ), ); feedsInfo[feedId.toString()] = { - value: Number(data.data.slice(0, 50)), + value: data.data, round: Number(data.index), }; } diff --git a/apps/e2e-tests/src/utils/services/sequencer.ts b/apps/e2e-tests/src/utils/services/sequencer.ts index 2086f319bb..288cf7df2c 100644 --- a/apps/e2e-tests/src/utils/services/sequencer.ts +++ b/apps/e2e-tests/src/utils/services/sequencer.ts @@ -104,19 +104,13 @@ export class Sequencer extends Context.Tag('@e2e-tests/Sequencer')< postReportsBatchUrl, reporterKey, getConfig: () => - Effect.gen(function* () { - return yield* fetchAndDecodeJSONEffect( - SequencerConfigV2Schema, - configUrl, - ).pipe(Effect.provide(FetchHttpClient.layer)); - }), + fetchAndDecodeJSONEffect(SequencerConfigV2Schema, configUrl).pipe( + Effect.provide(FetchHttpClient.layer), + ), getFeedsConfig: () => - Effect.gen(function* () { - return yield* fetchAndDecodeJSONEffect( - NewFeedsConfigSchema, - feedsConfigUrl, - ).pipe(Effect.provide(FetchHttpClient.layer)); - }), + fetchAndDecodeJSONEffect(NewFeedsConfigSchema, feedsConfigUrl).pipe( + Effect.provide(FetchHttpClient.layer), + ), fetchUpdatesToNetworksMetric: () => { return Effect.gen(function* () { const metrics = yield* getMetrics(metricsUrl).pipe( diff --git a/apps/e2e-tests/src/utils/services/types.ts b/apps/e2e-tests/src/utils/services/types.ts index e64c1795d1..0d5666ac27 100644 --- a/apps/e2e-tests/src/utils/services/types.ts +++ b/apps/e2e-tests/src/utils/services/types.ts @@ -1,5 +1,7 @@ import { Schema as S } from 'effect'; +import { hexDataString } from '@blocksense/base-utils'; + import type { FeedResult } from '../services/generate-signature'; export const UpdatesToNetworkMetric = S.Struct({ @@ -16,25 +18,24 @@ export const UpdatesToNetworkMetric = S.Struct({ ), }); -export type UpdatesToNetwork = Record>; - -const Numerical = S.Struct({ +export const Numerical = S.Struct({ Numerical: S.Number, }); +export type UpdatesToNetwork = Record>; + export const FeedAggregateHistorySchema = S.Struct({ aggregate_history: S.Record({ key: S.String, value: S.Array( S.Struct({ - value: Numerical, + value: S.Union(hexDataString, Numerical), update_number: S.Number, end_slot_timestamp: S.Number, }), ), }), }); - export type FeedAggregateHistory = typeof FeedAggregateHistorySchema.Type; export type ReportData = { diff --git a/apps/oracles/Cargo.lock b/apps/oracles/Cargo.lock index e60dabaa50..7ed3dc3d16 100644 --- a/apps/oracles/Cargo.lock +++ b/apps/oracles/Cargo.lock @@ -946,6 +946,8 @@ version = "0.1.1" dependencies = [ "anyhow", "blocksense-sdk", + "ethereum_ssz", + "ethereum_ssz_derive", "futures", "itertools 0.14.0", "serde", @@ -1562,6 +1564,46 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "ethereum_serde_utils" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc1355dbb41fbbd34ec28d4fb2a57d9a70c67ac3c19f6a5ca4d4a176b9e997a" +dependencies = [ + "alloy-primitives", + "hex", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "ethereum_ssz" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dcddb2554d19cde19b099fadddde576929d7a4d0c1cd3512d1fd95cf174375c" +dependencies = [ + "alloy-primitives", + "ethereum_serde_utils", + "itertools 0.13.0", + "serde", + "serde_derive", + "smallvec 1.15.1", + "typenum", +] + +[[package]] +name = "ethereum_ssz_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a657b6b3b7e153637dc6bdc6566ad9279d9ee11a15b12cfb24a2e04360637e9f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "exsat-holdings" version = "0.1.0" @@ -2308,6 +2350,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -3742,6 +3793,24 @@ dependencies = [ "der", ] +[[package]] +name = "sports-db" +version = "0.1.0" +dependencies = [ + "anyhow", + "blocksense-data-providers-sdk", + "blocksense-sdk", + "futures", + "serde", + "serde-this-or-that", + "serde_derive", + "serde_json", + "tracing", + "tracing-subscriber", + "url", + "wit-bindgen", +] + [[package]] name = "spout-rwa" version = "0.1.0" diff --git a/apps/oracles/Cargo.toml b/apps/oracles/Cargo.toml index 02ca0cad92..c804ef246c 100644 --- a/apps/oracles/Cargo.toml +++ b/apps/oracles/Cargo.toml @@ -8,6 +8,7 @@ members = [ "exsat-holdings", "forex-price-feeds", "gecko-terminal", + "sports-db", "spout-rwa", "stock-price-feeds", ] diff --git a/apps/oracles/sports-db/.gitignore b/apps/oracles/sports-db/.gitignore new file mode 100644 index 0000000000..1bd0e86768 --- /dev/null +++ b/apps/oracles/sports-db/.gitignore @@ -0,0 +1,2 @@ +.spin/ +/target/ diff --git a/apps/oracles/sports-db/Cargo.toml b/apps/oracles/sports-db/Cargo.toml new file mode 100644 index 0000000000..239d340910 --- /dev/null +++ b/apps/oracles/sports-db/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "sports-db" +authors = ["Aneta Tsvetkova"] +description = "Oracle script providing sports data." +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +blocksense-data-providers-sdk = { workspace = true } +blocksense-sdk = { workspace = true } + +anyhow = { workspace = true } +futures = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde-this-or-that = { workspace = true } +serde_derive = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = { workspace = true } +wit-bindgen = { workspace = true } diff --git a/apps/oracles/sports-db/spin.toml b/apps/oracles/sports-db/spin.toml new file mode 100644 index 0000000000..754c61cec8 --- /dev/null +++ b/apps/oracles/sports-db/spin.toml @@ -0,0 +1,32 @@ +spin_manifest_version = 2 + +[application] +name = "Blocksense runtime" +version = "0.1.0" +authors = ["blocksense-network"] + +[application.trigger.settings] +interval_time_in_seconds = 10 +reporter_id = 0 +sequencer = "http://127.0.0.1:9856/post_reports_batch" +secret_key = "536d1f9d97166eba5ff0efb8cc8dbeb856fb13d2d126ed1efc761e9955014003" +second_consensus_secret_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +kafka_endpoint = "http://127.0.0.1:9092" +metrics_url = "http://127.0.0.1:9091" + +[[trigger.oracle]] +component = "sports-db" +capabilities = [] + +[[trigger.oracle.data_feeds]] +id = "0" +stride = 4 +decimals = 0 +data = '{"pair":null,"decimals":0,"category":"Sports","market_hours":null,"arguments":{"kind":"sports-db","sport_type":"Soccer","team_id":133602}}' + +[component.sports-db] +source = "../target/wasm32-wasip1/release/sports_db.wasm" +allowed_outbound_hosts = ["https://www.thesportsdb.com/"] + +[component.sports-db.build] +command = "cargo build --target wasm32-wasip1 --release" diff --git a/apps/oracles/sports-db/src/fetch_results.rs b/apps/oracles/sports-db/src/fetch_results.rs new file mode 100644 index 0000000000..ac0fcf661c --- /dev/null +++ b/apps/oracles/sports-db/src/fetch_results.rs @@ -0,0 +1,39 @@ +use anyhow::Result; +use blocksense_data_providers_sdk::sports_data::{ + fetchers::{events::last_team_event::LastTeamEventFetcher, fetch::fetch_all_results}, + traits::sports_fetcher::fetch, + types::{SportsResultData, SportsResults}, +}; +use futures::stream::FuturesUnordered; + +use crate::FeedConfig; + +pub async fn get_results(resources: &Vec, timeout_secs: u64) -> Result { + let futures_set = FuturesUnordered::from_iter(resources.iter().map(|resource| { + fetch::( + resource.arguments.team_id, + resource.arguments.sport_type.clone(), + None, + timeout_secs, + ) + })); + + let fetched_results = fetch_all_results(futures_set).await; + + let mut final_results = SportsResults::new(); + for price_data_for_exchange in fetched_results { + fill_results(&resources, price_data_for_exchange, &mut final_results); + } + Ok(final_results) +} + +fn fill_results( + resources: &Vec, + prices_per_exchange: SportsResultData, + results: &mut SportsResults, +) { + for resource in resources { + let res = results.entry(resource.feed_id).or_default(); + res.extend(prices_per_exchange.data.clone()); + } +} diff --git a/apps/oracles/sports-db/src/lib.rs b/apps/oracles/sports-db/src/lib.rs new file mode 100644 index 0000000000..600fdb963f --- /dev/null +++ b/apps/oracles/sports-db/src/lib.rs @@ -0,0 +1,80 @@ +mod fetch_results; +mod logging; + +use anyhow::Result; + +use blocksense_data_providers_sdk::sports_data::types::SportsResults; +use serde::{Deserialize, Serialize}; +use tracing::info; + +use blocksense_sdk::{ + oracle::{DataFeedResult, Payload, Settings}, + oracle_component, +}; + +use crate::{fetch_results::get_results, logging::print_payload}; + +pub type FeedId = u128; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FeedArguments { + pub team_id: u64, + pub sport_type: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FeedConfig { + #[serde(default, rename = "id")] + pub feed_id: FeedId, + pub arguments: FeedArguments, +} + +#[oracle_component] +async fn oracle_request(settings: Settings) -> Result { + tracing_subscriber::fmt::init(); + + info!("Starting oracle component"); + + let timeout_secs = settings.interval_time_in_seconds - 1; + let resources = get_resources_from_settings(&settings)?; + + let results = get_results(&resources, timeout_secs).await?; + let payload = process_results(results)?; + + print_payload(&payload, &resources); + + Ok(payload) +} + +fn process_results(results: SportsResults) -> Result { + let mut payload = Payload::new(); + for (feed_id, data) in results { + let data_feed_result = DataFeedResult { + id: feed_id.to_string(), + value: blocksense_sdk::oracle::DataFeedResultValue::Bytes(data), + }; + + payload.values.push(data_feed_result); + } + + Ok(payload) +} + +pub fn get_resources_from_settings(settings: &Settings) -> Result> { + let mut config: Vec = Vec::new(); + for feed_setting in &settings.data_feeds { + match serde_json::from_str::(&feed_setting.data) { + Ok(mut feed_config) => { + feed_config.feed_id = feed_setting.id.parse::()?; + config.push(feed_config); + } + Err(err) => { + println!( + "Error {err} when parsing feed settings data = '{}'", + &feed_setting.data + ); + } + } + } + Ok(config) +} diff --git a/apps/oracles/sports-db/src/logging.rs b/apps/oracles/sports-db/src/logging.rs new file mode 100644 index 0000000000..605c05eaec --- /dev/null +++ b/apps/oracles/sports-db/src/logging.rs @@ -0,0 +1,22 @@ +use blocksense_sdk::oracle::{ + logging::{print_oracle_results, LoggingConfig, ResourceLogEntry}, + Payload, +}; + +use crate::FeedConfig; + +impl ResourceLogEntry for FeedConfig { + fn get_id_str(&self) -> String { + self.feed_id.to_string() + } + fn get_display_name(&self) -> String { + format!( + "{} Sports Results for Team ID: {} (Sport Type: {})", + self.feed_id, self.arguments.team_id, self.arguments.sport_type + ) + } +} + +pub fn print_payload(payload: &Payload, resources: &[FeedConfig]) { + print_oracle_results(resources, payload, LoggingConfig::basic()); +} diff --git a/apps/sequencer/src/feeds/feed_config_conversions.rs b/apps/sequencer/src/feeds/feed_config_conversions.rs index 13cd8fb2f4..2cd996c1fc 100644 --- a/apps/sequencer/src/feeds/feed_config_conversions.rs +++ b/apps/sequencer/src/feeds/feed_config_conversions.rs @@ -14,10 +14,12 @@ pub fn feed_config_to_block(feed_config: &FeedConfig) -> BlockFeedConfig { description: string_to_data_chunk(&feed_config.description), _type: string_to_data_chunk(&feed_config.feed_type), decimals: feed_config.additional_feed_info.decimals, - pair: blocksense_blockchain_data_model::AssetPair { - base: string_to_data_chunk(feed_config.additional_feed_info.pair.base.as_str()), - quote: string_to_data_chunk(feed_config.additional_feed_info.pair.quote.as_str()), - }, + pair: feed_config.additional_feed_info.pair.as_ref().map(|pair| { + blocksense_blockchain_data_model::AssetPair { + base: string_to_data_chunk(pair.base.as_str()), + quote: string_to_data_chunk(pair.quote.as_str()), + } + }), report_interval_ms: feed_config.schedule.interval_ms, first_report_start_time: feed_config.schedule.first_report_start_unix_time_ms, resources: json_to_byte_arrays(&feed_config.additional_feed_info.arguments) @@ -100,10 +102,10 @@ pub fn block_feed_to_feed_config(block_feed: &BlockFeedConfig) -> FeedConfig { first_report_start_unix_time_ms: block_feed.first_report_start_time, }, additional_feed_info: PriceFeedInfo { - pair: AssetPair { - base: data_chunk_to_string(&block_feed.pair.base), - quote: data_chunk_to_string(&block_feed.pair.quote), - }, + pair: block_feed.pair.as_ref().map(|pair| AssetPair { + base: data_chunk_to_string(&pair.base), + quote: data_chunk_to_string(&pair.quote), + }), decimals: block_feed.decimals, category: "".to_string(), market_hours: None, diff --git a/apps/trigger-oracle/src/lib.rs b/apps/trigger-oracle/src/lib.rs index b3059dd499..fb926e55af 100644 --- a/apps/trigger-oracle/src/lib.rs +++ b/apps/trigger-oracle/src/lib.rs @@ -59,7 +59,7 @@ use blocksense_metrics::{ }, TextEncoder, }; -use blocksense_utils::{time::current_unix_time, EncodedFeedId, FeedId}; +use blocksense_utils::{time::current_unix_time, EncodedFeedId, FeedId, Stride}; use blocksense_gnosis_safe::{ data_types::{ConsensusSecondRoundBatch, ReporterResponse}, @@ -327,6 +327,12 @@ impl TriggerExecutor for OracleTrigger { ); } } + let feed_stride_defaults: Arc> = Arc::new( + feeds_config + .keys() + .map(|encoded_id| (encoded_id.get_id(), encoded_id.get_stride())) + .collect(), + ); let mut data_feed_senders = HashMap::new(); tracing::trace!("Starting oracle scripts"); let mut loops: Vec<_> = self @@ -379,6 +385,7 @@ impl TriggerExecutor for OracleTrigger { &sequencer_post_batch_url, &self.secret_key, self.reporter_id, + feed_stride_defaults.clone(), ); loops.push(manager); @@ -404,17 +411,18 @@ impl TriggerExecutor for OracleTrigger { } impl OracleTrigger { - fn format_feed_id(raw_id: &str) -> String { - if let Some(encoded) = Self::parse_encoded_feed_id(raw_id) { + fn format_feed_id(raw_id: &str, default_strides: &HashMap) -> String { + if let Some(encoded) = Self::parse_encoded_feed_id(raw_id, default_strides) { format!("{}:{}", encoded.get_stride(), encoded.get_id()) - } else if let Ok(feed) = raw_id.parse::() { - format!("0:{feed}") } else { format!("0:{raw_id}") } } - fn parse_encoded_feed_id(feed_id: &str) -> Option { + fn parse_encoded_feed_id( + feed_id: &str, + default_strides: &HashMap, + ) -> Option { let mut parts = feed_id.split(':'); let first = parts.next()?; let second = parts.next(); @@ -427,10 +435,11 @@ impl OracleTrigger { None } } - None => first - .parse::() - .ok() - .and_then(|feed| EncodedFeedId::try_new(feed, 0)), + None => { + let feed_id = first.parse::().ok()?; + let stride = default_strides.get(&feed_id).copied().unwrap_or(0); + EncodedFeedId::try_new(feed_id, stride) + } } } @@ -737,6 +746,7 @@ impl OracleTrigger { sequencer_post_batch_url: &Url, secret_key: &str, reporter_id: u64, + feed_stride_defaults: Arc>, ) -> JoinHandle { let process_payload_future = Self::process_payload( payload_rx, @@ -744,6 +754,7 @@ impl OracleTrigger { sequencer_post_batch_url.to_owned(), secret_key.to_owned(), reporter_id, + feed_stride_defaults, ); spawn(process_payload_future) @@ -755,6 +766,7 @@ impl OracleTrigger { sequencer_url: Url, secret_key: String, reporter_id: u64, + feed_stride_defaults: Arc>, ) -> TerminationReason { tracing::trace!("Task sender to sequencer started"); while let Some((_component_id, payload)) = rx.recv().await { @@ -768,6 +780,7 @@ impl OracleTrigger { let result = match value { oracle::DataFeedResultValue::Numerical(value) => Ok(FeedType::Numerical(value)), oracle::DataFeedResultValue::Text(value) => Ok(FeedType::Text(value)), + oracle::DataFeedResultValue::Bytes(value) => Ok(FeedType::Bytes(value)), oracle::DataFeedResultValue::Error(error_string) => { Err(FeedError::APIError(error_string)) } @@ -778,7 +791,7 @@ impl OracleTrigger { } }; - let feed_id = Self::format_feed_id(&id); + let feed_id = Self::format_feed_id(&id, &feed_stride_defaults); let signature = generate_signature(&secret_key, feed_id.as_str(), timestamp, &result).unwrap(); @@ -1119,7 +1132,8 @@ fn update_latest_votes( ) { for vote in batch { let Some(encoded_feed_id) = - OracleTrigger::parse_encoded_feed_id(&vote.payload_metadata.feed_id) + OracleTrigger::parse_encoded_feed_id(&vote.payload_metadata.feed_id, &HashMap::new()) + // TODO: pass the actual strides from config for second round consensus else { tracing::warn!( "Failed to parse feed id '{}' when updating latest votes", diff --git a/libs/blockchain_data_model/src/lib.rs b/libs/blockchain_data_model/src/lib.rs index 05c7f311b0..0ed3873980 100644 --- a/libs/blockchain_data_model/src/lib.rs +++ b/libs/blockchain_data_model/src/lib.rs @@ -25,7 +25,7 @@ pub struct BlockFeedConfig { pub description: DataChunk, pub _type: DataChunk, pub decimals: u8, - pub pair: AssetPair, + pub pair: Option, pub report_interval_ms: u64, pub first_report_start_time: u64, pub resources: Resources, diff --git a/libs/config/src/lib.rs b/libs/config/src/lib.rs index ec9661a15f..27c1740ca1 100644 --- a/libs/config/src/lib.rs +++ b/libs/config/src/lib.rs @@ -497,10 +497,10 @@ pub fn test_feed_config(id: FeedId, stride: u8) -> FeedConfig { .as_millis() as u64, }, additional_feed_info: PriceFeedInfo { - pair: blocksense_registry::config::AssetPair { + pair: Some(blocksense_registry::config::AssetPair { base: "FOXY".to_owned(), quote: "USD".to_owned(), - }, + }), decimals: 18, category: "Crypto".to_owned(), market_hours: Some("Crypto".to_owned()), diff --git a/libs/feed_registry/src/aggregate.rs b/libs/feed_registry/src/aggregate.rs index 492ae0a073..0ecd88099d 100644 --- a/libs/feed_registry/src/aggregate.rs +++ b/libs/feed_registry/src/aggregate.rs @@ -39,26 +39,84 @@ impl FeedAggregate { let span = info_span!("MajorityVoteAggregator"); let _guard = span.enter(); - let mut frequency_map = HashMap::new(); + let mut frequency_map_text = HashMap::new(); + let mut frequency_map_bytes = HashMap::new(); // Count the occurrences of each string for v in values { match v { - FeedType::Text(t) => *frequency_map.entry(t).or_insert(0) += 1, + FeedType::Text(t) => *frequency_map_text.entry(t).or_insert(0) += 1, + FeedType::Bytes(b) => *frequency_map_bytes.entry(b).or_insert(0) += 1, _ => { - error!("Attempting to perform frequency_map on f64!"); + error!("Attempting to perform frequency_map on {:?}", v); } } } // Find the string with the maximum occurrences - let result = frequency_map - .into_iter() - .max_by_key(|&(_, count)| count) - .map(|(s, _)| s) - .expect("Aggregating empty set of values!") - .clone(); - FeedType::Text(result) + if !frequency_map_text.is_empty() { + assert!( + frequency_map_bytes.is_empty(), + "Mixing Text and Bytes in MajorityVoteAggregator is not allowed!" + ); + let result = frequency_map_text + .into_iter() + .max_by_key(|&(_, count)| count) + .map(|(s, _)| s) + .unwrap() + .clone(); + FeedType::Text(result) + } else if !frequency_map_bytes.is_empty() { + let most_frequent = frequency_map_bytes + .into_iter() + .max_by_key(|&(_, count)| count) + .map(|(s, _)| s) + .unwrap() + .clone(); + + fn prefix_size(stride: u8) -> u8 { + // `2 ^ (n + 1)` divided by 8, rounded up + (stride + 5).div_ceil(8) + } + + fn right_align_truncate(dst: &mut [u8], src: &[u8]) { + let dst_len = dst.len(); + let src_len = src.len(); + + let n = std::cmp::min(dst_len, src_len); + dst[dst_len - n..].copy_from_slice(&src[src_len - n..]); + } + + let prefix_size = prefix_size(4) as usize; + let mut prefix = vec![0; prefix_size]; + + let length = most_frequent.len(); + + // length | prefix + // (in bytes) | + // ------------------------------------ + // 0x 00 .. 00 12 34 | 0x 00 00 00 00 + + // NOTE: Zip together the pointers to the bytes of the prefix and the length, starting from the "right" + // std::iter::zip(prefix[..].iter_mut().rev(), length.to_le_bytes()).for_each( + // |(t, s)| { + // *t = s; + // }, + // ); + right_align_truncate(&mut prefix, &(length + prefix_size).to_be_bytes()); + + let mut result = [prefix, most_frequent].concat(); + + // Calculate max SSZ slots and pad with zeros + let max_ssz_slots = (result.len() - 2).div_ceil(64); + let target_length = max_ssz_slots * 64 + 2; + + result.resize(target_length, 0); + + FeedType::Bytes(result) + } else { + panic!("No valid types to aggregate in MajorityVoteAggregator!"); + } } FeedAggregate::MedianAggregator => { let span = info_span!("MedianAggregator"); diff --git a/libs/registry/src/config.rs b/libs/registry/src/config.rs index c2d1822b92..7a2bb94e14 100644 --- a/libs/registry/src/config.rs +++ b/libs/registry/src/config.rs @@ -43,7 +43,7 @@ pub struct FeedSchedule { #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] pub struct PriceFeedInfo { - pub pair: AssetPair, + pub pair: Option, pub decimals: u8, pub category: String, pub market_hours: Option, diff --git a/libs/sdk/Cargo.lock b/libs/sdk/Cargo.lock index 43c0a59c60..4a97d3b99f 100644 --- a/libs/sdk/Cargo.lock +++ b/libs/sdk/Cargo.lock @@ -618,7 +618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e64c09ec565a90ed8390d82aa08cd3b22e492321b96cb4a3d4f58414683c9e2f" dependencies = [ "alloy-primitives", - "darling", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.106", @@ -990,6 +990,8 @@ version = "0.1.1" dependencies = [ "anyhow", "blocksense-sdk", + "ethereum_ssz", + "ethereum_ssz_derive", "futures", "itertools 0.14.0", "serde", @@ -1250,14 +1252,38 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.106", ] [[package]] @@ -1275,13 +1301,24 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.106", +] + [[package]] name = "darling_macro" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core", + "darling_core 0.21.3", "quote", "syn 2.0.106", ] @@ -1524,6 +1561,46 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "ethereum_serde_utils" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc1355dbb41fbbd34ec28d4fb2a57d9a70c67ac3c19f6a5ca4d4a176b9e997a" +dependencies = [ + "alloy-primitives", + "hex", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "ethereum_ssz" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dcddb2554d19cde19b099fadddde576929d7a4d0c1cd3512d1fd95cf174375c" +dependencies = [ + "alloy-primitives", + "ethereum_serde_utils", + "itertools 0.13.0", + "serde", + "serde_derive", + "smallvec", + "typenum", +] + +[[package]] +name = "ethereum_ssz_derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a657b6b3b7e153637dc6bdc6566ad9279d9ee11a15b12cfb24a2e04360637e9f" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -3448,7 +3525,7 @@ version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7e6c180db0816026a61afa1cff5344fb7ebded7e4d3062772179f2501481c27" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.106", diff --git a/libs/sdk/Cargo.toml b/libs/sdk/Cargo.toml index aab85b5355..6587153c4d 100644 --- a/libs/sdk/Cargo.toml +++ b/libs/sdk/Cargo.toml @@ -16,6 +16,8 @@ blocksense-sdk = { path = "./sdk" } alloy = "1.0.9" anyhow = "1" async-trait = "0.1" +ethereum_ssz = "0.9.1" +ethereum_ssz_derive = "0.9.1" futures = "0.3" itertools = "0.14.0" prettytable-rs = "0.10.0" diff --git a/libs/sdk/data_providers/Cargo.toml b/libs/sdk/data_providers/Cargo.toml index d377aa66c2..608c267e2b 100644 --- a/libs/sdk/data_providers/Cargo.toml +++ b/libs/sdk/data_providers/Cargo.toml @@ -12,6 +12,8 @@ name = "blocksense_data_providers_sdk" blocksense-sdk = { workspace = true } anyhow = { workspace = true } +ethereum_ssz = { workspace = true } +ethereum_ssz_derive = { workspace = true } futures = { workspace = true } itertools = { workspace = true } serde = { workspace = true } diff --git a/libs/sdk/data_providers/src/lib.rs b/libs/sdk/data_providers/src/lib.rs index 581004cad1..98d91de8c8 100644 --- a/libs/sdk/data_providers/src/lib.rs +++ b/libs/sdk/data_providers/src/lib.rs @@ -1 +1,2 @@ pub mod price_data; +pub mod sports_data; diff --git a/libs/sdk/data_providers/src/sports_data/fetchers/events/last_team_event.rs b/libs/sdk/data_providers/src/sports_data/fetchers/events/last_team_event.rs new file mode 100644 index 0000000000..47dd32a13b --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/fetchers/events/last_team_event.rs @@ -0,0 +1,128 @@ +use std::collections::HashMap; + +use anyhow::{Context, Result}; +use futures::{future::LocalBoxFuture, FutureExt}; + +use serde::Deserialize; + +use blocksense_sdk::http::http_get_json; +use ssz::Encode; +use ssz_derive::Encode; + +use crate::sports_data::traits::sports_fetcher::SportsFetcher; + +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LastEventData { + #[serde(rename = "idEvent", deserialize_with = "deserialize_u64_from_str")] + pub event_id: u64, +} + +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LastEvent { + pub results: Vec, +} + +type LastEventResponse = LastEvent; + +fn deserialize_utf8_bytes<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + Ok(s.into_bytes()) +} + +fn deserialize_u64_from_str<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + s.parse().map_err(serde::de::Error::custom) +} + +fn deserialize_to_le_u64_from_str<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + let value: u64 = s.parse().map_err(serde::de::Error::custom)?; + Ok(value.to_be()) +} + +#[derive(Default, Debug, Clone, PartialEq, Deserialize, Encode)] +#[serde(rename_all = "camelCase")] +pub struct EventLookupData { + #[serde(rename = "strEvent", deserialize_with = "deserialize_utf8_bytes")] + pub event_name: Vec, + + #[serde(rename = "strSeason", deserialize_with = "deserialize_utf8_bytes")] + pub season: Vec, + + #[serde(rename = "strHomeTeam", deserialize_with = "deserialize_utf8_bytes")] + pub home_team: Vec, + + #[serde(rename = "strAwayTeam", deserialize_with = "deserialize_utf8_bytes")] + pub away_team: Vec, + + #[serde( + rename = "intHomeScore", + deserialize_with = "deserialize_to_le_u64_from_str" + )] + pub home_score: u64, + + #[serde( + rename = "intAwayScore", + deserialize_with = "deserialize_to_le_u64_from_str" + )] + pub away_score: u64, +} + +#[derive(Default, Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EventLookup { + pub events: Vec, +} + +type EventLookupResponse = EventLookup; + +pub struct LastTeamEventFetcher { + pub id: u64, +} + +impl SportsFetcher for LastTeamEventFetcher { + fn new(id: u64, _api_keys: Option>) -> Self { + Self { id } + } + + fn fetch(&self, timeout_secs: u64) -> LocalBoxFuture<'_, Result>> { + async move { + let response = http_get_json::( + "https://www.thesportsdb.com/api/v1/json/123/eventslast.php", + Some(&[("id", &self.id.to_string())]), + None, + Some(timeout_secs), + ) + .await?; + + // Return the first event data if available + let first_result = response.results.first().context("No event results found")?; + let response = http_get_json::( + "https://www.thesportsdb.com/api/v1/json/123/lookupevent.php", + Some(&[("id", &first_result.event_id.to_string())]), + None, + Some(timeout_secs), + ) + .await?; + + // Return the first event data if available + if let Some(first_event) = response.events.first() { + Ok(first_event.as_ssz_bytes()) + } else { + Err(anyhow::anyhow!("No event data found for the given ID")) + } + } + .boxed_local() + } +} diff --git a/libs/sdk/data_providers/src/sports_data/fetchers/events/mod.rs b/libs/sdk/data_providers/src/sports_data/fetchers/events/mod.rs new file mode 100644 index 0000000000..a7e85239d2 --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/fetchers/events/mod.rs @@ -0,0 +1 @@ +pub mod last_team_event; diff --git a/libs/sdk/data_providers/src/sports_data/fetchers/fetch.rs b/libs/sdk/data_providers/src/sports_data/fetchers/fetch.rs new file mode 100644 index 0000000000..80fdc84cae --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/fetchers/fetch.rs @@ -0,0 +1,36 @@ +use std::time::Instant; + +use anyhow::Result; +use futures::stream::StreamExt; +use futures::Stream; +use tracing::{info, warn}; + +use crate::sports_data::types::SportsResultData; + +pub async fn fetch_all_results(mut futures_set: S) -> Vec +where + S: Stream>)> + Unpin, +{ + let mut all_fetched_results = Vec::new(); + let before_fetch = Instant::now(); + + // Process results as they complete + while let Some((sport_type, result)) = futures_set.next().await { + match result { + Ok(ssz_encoded_result) => { + let time_taken = before_fetch.elapsed(); + info!("â„šī¸ Successfully fetched results for {sport_type} in {time_taken:?}"); + let result_data = SportsResultData { + sport_type: sport_type.to_owned(), + data: ssz_encoded_result, + }; + all_fetched_results.push(result_data); + } + Err(err) => warn!("❌ Error fetching results for {sport_type}: {err:?}"), + } + } + + info!("🕛 All results fetched in {:?}", before_fetch.elapsed()); + + all_fetched_results +} diff --git a/libs/sdk/data_providers/src/sports_data/fetchers/mod.rs b/libs/sdk/data_providers/src/sports_data/fetchers/mod.rs new file mode 100644 index 0000000000..d1812509ca --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/fetchers/mod.rs @@ -0,0 +1,2 @@ +pub mod events; +pub mod fetch; diff --git a/libs/sdk/data_providers/src/sports_data/mod.rs b/libs/sdk/data_providers/src/sports_data/mod.rs new file mode 100644 index 0000000000..38aac42d03 --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/mod.rs @@ -0,0 +1,3 @@ +pub mod fetchers; +pub mod traits; +pub mod types; diff --git a/libs/sdk/data_providers/src/sports_data/traits/mod.rs b/libs/sdk/data_providers/src/sports_data/traits/mod.rs new file mode 100644 index 0000000000..7a02c72abb --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/traits/mod.rs @@ -0,0 +1 @@ +pub mod sports_fetcher; diff --git a/libs/sdk/data_providers/src/sports_data/traits/sports_fetcher.rs b/libs/sdk/data_providers/src/sports_data/traits/sports_fetcher.rs new file mode 100644 index 0000000000..a4267a1968 --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/traits/sports_fetcher.rs @@ -0,0 +1,27 @@ +use std::collections::HashMap; + +use anyhow::Result; +use futures::future::LocalBoxFuture; +use futures::FutureExt; + +pub trait SportsFetcher { + fn new(id: u64, api_keys: Option>) -> Self; + fn fetch(&self, timeout_secs: u64) -> LocalBoxFuture<'_, Result>>; +} + +pub fn fetch<'a, PF>( + id: u64, + sport_type: String, + api_keys: Option>, + timeout_secs: u64, +) -> LocalBoxFuture<'a, (String, Result>)> +where + PF: SportsFetcher, +{ + async move { + let fetcher = PF::new(id, api_keys); + let res = fetcher.fetch(timeout_secs).await; + (sport_type, res) + } + .boxed_local() +} diff --git a/libs/sdk/data_providers/src/sports_data/types.rs b/libs/sdk/data_providers/src/sports_data/types.rs new file mode 100644 index 0000000000..942ff7a420 --- /dev/null +++ b/libs/sdk/data_providers/src/sports_data/types.rs @@ -0,0 +1,11 @@ +use std::collections::HashMap; + +pub type SportType = String; + +#[derive(Clone, Debug)] +pub struct SportsResultData { + pub sport_type: SportType, + pub data: Vec, +} + +pub type SportsResults = HashMap>; diff --git a/libs/sdk/macro/src/lib.rs b/libs/sdk/macro/src/lib.rs index 4ee514df0f..32e885e4f4 100644 --- a/libs/sdk/macro/src/lib.rs +++ b/libs/sdk/macro/src/lib.rs @@ -82,6 +82,7 @@ pub fn oracle_component(_attr: TokenStream, item: TokenStream) -> TokenStream { ::blocksense_sdk::oracle::DataFeedResultValue::None => Self::None, ::blocksense_sdk::oracle::DataFeedResultValue::Numerical(value) => Self::Numerical(value), ::blocksense_sdk::oracle::DataFeedResultValue::Text(value) => Self::Text(value), + ::blocksense_sdk::oracle::DataFeedResultValue::Bytes(value) => Self::Bytes(value), ::blocksense_sdk::oracle::DataFeedResultValue::Error(error) => Self::Error(error), } } diff --git a/libs/sdk/sdk/src/oracle.rs b/libs/sdk/sdk/src/oracle.rs index abee6bb8ab..3ac3297ac7 100644 --- a/libs/sdk/sdk/src/oracle.rs +++ b/libs/sdk/sdk/src/oracle.rs @@ -41,6 +41,7 @@ pub enum DataFeedResultValue { None, Numerical(f64), Text(String), + Bytes(Vec), Error(String), } diff --git a/libs/sdk/sdk/src/oracle/logging.rs b/libs/sdk/sdk/src/oracle/logging.rs index 7008b4abdb..ff9bd8812a 100644 --- a/libs/sdk/sdk/src/oracle/logging.rs +++ b/libs/sdk/sdk/src/oracle/logging.rs @@ -1,3 +1,4 @@ +use alloy::hex; use prettytable::{format, Cell, Row, Table}; use tracing::{info, warn}; @@ -99,7 +100,10 @@ where .find(|x| x.id == id_str) .and_then(|v| match &v.value { DataFeedResultValue::Numerical(num) => Some(format!("{num:.8}")), - _ => None, + DataFeedResultValue::Bytes(bytes) => Some(format!("0x{}", hex::encode(bytes))), + DataFeedResultValue::Text(text) => Some(text.clone()), + DataFeedResultValue::Error(err) => Some(format!("Error: {}", err)), + DataFeedResultValue::None => None, }) .unwrap_or_else(|| { track_missing_price(&id_str, &resource.get_display_name(), &providers) diff --git a/libs/sdk/wit/blocksense-oracle.wit b/libs/sdk/wit/blocksense-oracle.wit index b534500488..44e1f91703 100644 --- a/libs/sdk/wit/blocksense-oracle.wit +++ b/libs/sdk/wit/blocksense-oracle.wit @@ -21,6 +21,7 @@ interface oracle-types { none, error(string), numerical(f64), + bytes(list), text(string), } @@ -30,7 +31,7 @@ interface oracle-types { } record payload { - values: list + values: list, } variant error { diff --git a/libs/ts/config-types/src/data-feeds-config/oracles.ts b/libs/ts/config-types/src/data-feeds-config/oracles.ts index 82e02dfcb6..f11be127a1 100644 --- a/libs/ts/config-types/src/data-feeds-config/oracles.ts +++ b/libs/ts/config-types/src/data-feeds-config/oracles.ts @@ -128,3 +128,12 @@ export const ethGasInfoArgsSchema = S.mutable( metric: S.String, }), ); + +// `sports-db` Oracle related Types +export const sportsArgsSchema = S.mutable( + S.Struct({ + kind: S.Literal('sports-db'), + team_id: S.Int, + sport_type: S.Union(S.Literal('Soccer')), + }), +); diff --git a/libs/ts/config-types/src/data-feeds-config/types.ts b/libs/ts/config-types/src/data-feeds-config/types.ts index a22e7942d1..a31b150095 100644 --- a/libs/ts/config-types/src/data-feeds-config/types.ts +++ b/libs/ts/config-types/src/data-feeds-config/types.ts @@ -9,6 +9,7 @@ import { forexPriceFeedsArgsSchema, geckoTerminalArgsSchema, hyperBorrowRatesArgsSchema, + sportsArgsSchema, spoutRwaArgsSchema, stockPriceFeedsArgsSchema, } from './oracles'; @@ -29,6 +30,7 @@ export const FeedCategorySchema = S.Union( S.Literal('US Treasuries'), S.Literal('Tokenized Asset'), S.Literal('Rates'), + S.Literal('Sports'), ).annotations({ identifier: 'FeedCategory' }); /** @@ -129,7 +131,10 @@ export const decodeFeedsConfig = S.decodeUnknownSync(FeedsConfigSchema); /** * Schema for the data feed type. */ -export const FeedTypeSchema = S.Union(S.Literal('price-feed')).annotations({ +export const FeedTypeSchema = S.Union( + S.Literal('price-feed'), + S.Literal('sport-feed'), +).annotations({ identifier: 'FeedType', }); @@ -174,9 +179,7 @@ export const NewFeedSchema = S.mutable( full_name: S.String, description: S.String, - type: S.Union(S.Literal('price-feed')).annotations({ - identifier: 'FeedType', - }), + type: FeedTypeSchema, oracle_id: S.String, value_type: S.Union( @@ -192,7 +195,11 @@ export const NewFeedSchema = S.mutable( quorum: S.Struct({ percentage: S.Number, - aggregation: S.Union(S.Literal('median')).annotations({ + aggregation: S.Union( + S.Literal('median'), + S.Literal('average'), + S.Literal('majority'), + ).annotations({ identifier: 'QuorumAggregation', }), }).annotations({ identifier: 'FeedQuorum' }), @@ -207,7 +214,7 @@ export const NewFeedSchema = S.mutable( // TODO: This field should be optional / different depending on the `type`. additional_feed_info: S.mutable( S.Struct({ - pair: PairSchema, + pair: S.NullishOr(PairSchema), decimals: S.Number, category: FeedCategorySchema, market_hours: S.NullishOr(MarketHoursSchema), @@ -222,6 +229,7 @@ export const NewFeedSchema = S.mutable( exSatHoldingsArgsSchema, hyperBorrowRatesArgsSchema, ethGasInfoArgsSchema, + sportsArgsSchema, ).annotations({ identifier: 'OracleScriptArguments' }), compatibility_info: S.UndefinedOr( diff --git a/libs/ts/decoders/src/scripts/generate-decoders.ts b/libs/ts/decoders/src/scripts/generate-decoders.ts index 04be5ed6d4..0ea227ba53 100644 --- a/libs/ts/decoders/src/scripts/generate-decoders.ts +++ b/libs/ts/decoders/src/scripts/generate-decoders.ts @@ -14,6 +14,7 @@ export const generateDecoders = async ( subTemplatePath?: string; contractName?: string; containsUnion?: boolean; + stride?: number; }, ) => { const defaultOptions = { @@ -41,10 +42,24 @@ export const generateDecoders = async ( ? await fs.readFile(opts.subTemplatePath, 'utf-8') : ''; + // In how many bytes will the prefix length (size of the data) be stored + // 32 * 2^(stride) bytes + // e.g. stride 0 -> 32 bytes -> 1 prefix byte + // stride 2 -> 128 bytes -> 1 prefix byte + // stride 3 -> 256 bytes -> 2 prefix bytes + const prefixSize = opts.stride ? Math.ceil((opts.stride + 5) / 8) : undefined; + const code = type === 'encode-packed' ? await generateEPDecoder(template, fields, evmVersion) - : await generateSSZDecoder(template, subTemplateSSZ, fields, evmVersion); + : await generateSSZDecoder( + template, + subTemplateSSZ, + fields, + evmVersion, + prefixSize, + !!prefixSize, + ); if (typeof code === 'string') { contractPaths.push(path.join(outputDir, opts.contractName + '.sol')); diff --git a/libs/ts/decoders/src/ssz/index.ts b/libs/ts/decoders/src/ssz/index.ts index 43e8d183e1..6038d4aa71 100644 --- a/libs/ts/decoders/src/ssz/index.ts +++ b/libs/ts/decoders/src/ssz/index.ts @@ -22,6 +22,7 @@ export const generateDecoder = async ( fields: TupleField, evmVersion: EvmVersion = 'cancun', start: number = 0, + paddedLen: boolean = false, ): Promise> => { const { schema, unionTypes } = await sszSchema(fields); @@ -43,6 +44,7 @@ export const generateDecoder = async ( mainStructName, evmVersion, start, + paddedLen, ); let subDecoderGeneratedLines: string[] = []; if (unionTypes) { diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index 6cb1770848..5f6ae1ea71 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -83,6 +83,7 @@ commodities-price-feeds = mkOracleScript "commodities-price-feeds"; spout-rwa = mkOracleScript "spout-rwa"; eth-gas-info = mkOracleScript "eth-gas-info"; + sports-db = mkOracleScript "sports-db"; }; spinPlugins = { diff --git a/nix/shells/pkg-sets/dev-shell.nix b/nix/shells/pkg-sets/dev-shell.nix index 94d9d320ba..76a94ba9e4 100644 --- a/nix/shells/pkg-sets/dev-shell.nix +++ b/nix/shells/pkg-sets/dev-shell.nix @@ -16,6 +16,7 @@ just findutils process-compose + coreutils ripgrep ]; diff --git a/yarn.lock b/yarn.lock index 0bddbad4dd..f01fe687d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1569,7 +1569,7 @@ __metadata: languageName: unknown linkType: soft -"@blocksense/dev@workspace:apps/dev": +"@blocksense/dev@workspace:*, @blocksense/dev@workspace:apps/dev": version: 0.0.0-use.local resolution: "@blocksense/dev@workspace:apps/dev" dependencies: @@ -1686,6 +1686,7 @@ __metadata: "@blocksense/base-utils": "workspace:*" "@blocksense/config-types": "workspace:*" "@blocksense/contracts": "workspace:*" + "@blocksense/dev": "workspace:*" "@chainsafe/bls": "npm:^8.2.0" "@effect/cluster": "npm:^0.48.2" "@effect/experimental": "npm:^0.54.6" @@ -1701,6 +1702,7 @@ __metadata: parse-prometheus-text-format: "npm:^1.1.1" tsx: "npm:^4.20.5" typescript: "npm:5.9.2" + viem: "npm:^2.38.6" vitest: "npm:^3.2.4" languageName: unknown linkType: soft @@ -30417,6 +30419,27 @@ __metadata: languageName: node linkType: hard +"viem@npm:^2.38.6": + version: 2.38.6 + resolution: "viem@npm:2.38.6" + dependencies: + "@noble/curves": "npm:1.9.1" + "@noble/hashes": "npm:1.8.0" + "@scure/bip32": "npm:1.7.0" + "@scure/bip39": "npm:1.6.0" + abitype: "npm:1.1.0" + isows: "npm:1.0.7" + ox: "npm:0.9.6" + ws: "npm:8.18.3" + peerDependencies: + typescript: ">=5.0.4" + peerDependenciesMeta: + typescript: + optional: true + checksum: 10c0/9b8571bc9d7dfc414eb72700275ac71b9402cbf4fe3e447501d252cbc43cb7ddf9ce01a16c183fac265d071873d209efc2a459462c724624ef855d941d3447f0 + languageName: node + linkType: hard + "viem@npm:^2.7.15": version: 2.21.25 resolution: "viem@npm:2.21.25"