diff --git a/crates/fidc-core/src/data.rs b/crates/fidc-core/src/data.rs index a242321..65dbcd2 100644 --- a/crates/fidc-core/src/data.rs +++ b/crates/fidc-core/src/data.rs @@ -1318,6 +1318,8 @@ pub struct DataSet { execution_quote_dates: Arc>, order_book_depth_index: Arc>>, benchmark_by_date: Arc>, + market_series_by_symbol: Arc>>, + adjusted_close_series_by_symbol: Arc>>, market_series_by_symbol_id: Arc>>>, adjusted_close_series_by_symbol_id: Arc>>>, market_series_end_positions_by_calendar_index: Arc>, @@ -1707,68 +1709,44 @@ impl DataSet { .into_iter() .map(|instrument| (instrument.symbol.clone(), instrument)) .collect::>(); - let symbol_id_by_code = build_symbol_id_index( - &instruments, - &market_by_date, - &factor_by_date, - &candidate_by_date, - ); - let symbol_count = symbol_id_by_code.len(); - let mut symbol_by_id = vec![Arc::::from(""); symbol_count]; - for (symbol, symbol_id) in &symbol_id_by_code { - symbol_by_id[*symbol_id as usize] = Arc::::from(symbol.as_str()); - } - let mut instruments_by_symbol_id = vec![None; symbol_count]; - for (symbol, instrument) in &instruments { - if let Some(symbol_id) = symbol_id_by_code.get(symbol).copied() { - instruments_by_symbol_id[symbol_id as usize] = Some(instrument.clone()); - } - } - - let mut market_rows_by_symbol_id = (0..symbol_count) - .map(|_| Vec::<&DailyMarketSnapshot>::new()) - .collect::>(); + let mut market_rows_by_symbol = AHashMap::>::new(); for row in market_by_date.values().flatten() { - let symbol_id = *symbol_id_by_code - .get(row.symbol.as_str()) - .expect("market symbol missing from FIDC symbol index"); - market_rows_by_symbol_id[symbol_id as usize].push(row); + if let Some(rows) = market_rows_by_symbol.get_mut(row.symbol.as_str()) { + rows.push(row); + continue; + } + market_rows_by_symbol.insert(row.symbol.clone(), vec![row]); } - let market_series_by_symbol_id = market_rows_by_symbol_id + let market_rows_by_symbol = market_rows_by_symbol.into_iter().collect::>(); + let market_series_by_symbol = market_rows_by_symbol .into_par_iter() - .enumerate() - .map(|(symbol_id, rows)| { - if rows.is_empty() { - return None; - } - Some(Arc::new(SymbolPriceSeries::from_sorted_rows( - symbol_by_id[symbol_id].to_string(), - rows, - ))) + .map(|(symbol, rows)| { + let series = Arc::new(SymbolPriceSeries::from_sorted_rows(symbol.clone(), rows)); + (symbol, series) }) - .collect::>(); - let mut factor_rows_by_symbol_id = (0..symbol_count) - .map(|_| Vec::<&DailyFactorSnapshot>::new()) - .collect::>(); + .collect::>() + .into_iter() + .collect::>(); + let mut factor_rows_by_symbol = AHashMap::<&str, Vec<&DailyFactorSnapshot>>::new(); for row in factor_by_date.values().flatten() { - let symbol_id = *symbol_id_by_code - .get(row.symbol.as_str()) - .expect("factor symbol missing from FIDC symbol index"); - factor_rows_by_symbol_id[symbol_id as usize].push(row); + factor_rows_by_symbol + .entry(row.symbol.as_str()) + .or_default() + .push(row); } - let adjusted_close_series_by_symbol_id = market_series_by_symbol_id + let adjusted_close_series_by_symbol = market_series_by_symbol .par_iter() - .enumerate() - .map(|(symbol_id, market)| { - market.as_ref().and_then(|market| { - AdjustedCloseSeries::new( - market, - factor_rows_by_symbol_id[symbol_id].as_slice(), - ) - .map(Arc::new) - }) + .filter_map(|(symbol, market)| { + let factor_rows = factor_rows_by_symbol + .get(symbol.as_str()) + .map(Vec::as_slice) + .unwrap_or_default(); + AdjustedCloseSeries::new(market, factor_rows) + .map(|series| (symbol.clone(), Arc::new(series))) }) - .collect::>(); + .collect::>() + .into_iter() + .collect::>(); let factor_texts = factor_texts .into_iter() .filter_map(|mut item| { @@ -1786,6 +1764,22 @@ impl DataSet { .map(|item| ((item.date, item.symbol.clone(), item.field.clone()), item)) .collect::>(); + let symbol_id_by_code = build_symbol_id_index( + &instruments, + &market_by_date, + &factor_by_date, + &candidate_by_date, + ); + let mut symbol_by_id = vec![Arc::::from(""); symbol_id_by_code.len()]; + for (symbol, symbol_id) in &symbol_id_by_code { + symbol_by_id[*symbol_id as usize] = Arc::::from(symbol.as_str()); + } + let mut instruments_by_symbol_id = vec![None; symbol_id_by_code.len()]; + for (symbol, instrument) in &instruments { + if let Some(symbol_id) = symbol_id_by_code.get(symbol).copied() { + instruments_by_symbol_id[symbol_id as usize] = Some(instrument.clone()); + } + } let market_symbol_ids_by_date = build_group_symbol_ids(&market_by_date, &symbol_id_by_code, |item| { item.symbol.as_str() @@ -1815,6 +1809,18 @@ impl DataSet { &candidate_symbol_ids_by_date, symbol_id_by_code.len(), ); + let mut market_series_by_symbol_id = vec![None; symbol_id_by_code.len()]; + for (symbol, series) in &market_series_by_symbol { + if let Some(symbol_id) = symbol_id_by_code.get(symbol).copied() { + market_series_by_symbol_id[symbol_id as usize] = Some(Arc::clone(series)); + } + } + let mut adjusted_close_series_by_symbol_id = vec![None; symbol_id_by_code.len()]; + for (symbol, series) in &adjusted_close_series_by_symbol { + if let Some(symbol_id) = symbol_id_by_code.get(symbol).copied() { + adjusted_close_series_by_symbol_id[symbol_id as usize] = Some(Arc::clone(series)); + } + } let market_series_end_positions_by_calendar_index = build_calendar_series_end_positions(&market_series_by_symbol_id, &calendar); let execution_quotes_by_date = build_execution_quote_index(execution_quotes); @@ -1847,6 +1853,8 @@ impl DataSet { execution_quote_dates: Arc::new(execution_quote_dates), order_book_depth_index: Arc::new(order_book_depth_index), benchmark_by_date: Arc::new(benchmark_by_date), + market_series_by_symbol: Arc::new(market_series_by_symbol), + adjusted_close_series_by_symbol: Arc::new(adjusted_close_series_by_symbol), market_series_by_symbol_id: Arc::new(market_series_by_symbol_id), adjusted_close_series_by_symbol_id: Arc::new(adjusted_close_series_by_symbol_id), market_series_end_positions_by_calendar_index: Arc::new( @@ -1985,7 +1993,7 @@ impl DataSet { } fn market_series(&self, symbol: &str) -> Option<&SymbolPriceSeries> { - self.market_series_by_symbol_id(self.symbol_id(symbol)?) + self.market_series_by_symbol.get(symbol).map(Arc::as_ref) } fn market_series_by_symbol_id(&self, symbol_id: u32) -> Option<&SymbolPriceSeries> { @@ -1995,7 +2003,9 @@ impl DataSet { } fn adjusted_close_series(&self, symbol: &str) -> Option<&AdjustedCloseSeries> { - self.adjusted_close_series_by_symbol_id(self.symbol_id(symbol)?) + self.adjusted_close_series_by_symbol + .get(symbol) + .map(Arc::as_ref) } fn adjusted_close_series_by_symbol_id(&self, symbol_id: u32) -> Option<&AdjustedCloseSeries> { diff --git a/docs/evidence/symbol-id-series-storage-20260906.json b/docs/evidence/symbol-id-series-storage-20260906.json deleted file mode 100644 index ab74b99..0000000 --- a/docs/evidence/symbol-id-series-storage-20260906.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "schemaVersion": "fidc-symbol-id-series-storage/v1", - "measuredAt": "2026-09-06T06:25:00+08:00", - "host": "192.168.31.177", - "engineCommit": "5a7c49a", - "serviceCommit": "61a478839674ab03356083fb777840342ad4c726", - "implementation": { - "description": "build market and adjusted-close series directly in immutable symbol-id vectors and route string helpers through the canonical symbol-id index", - "removedStructures": [ - "market_series_by_symbol AHashMap", - "adjusted_close_series_by_symbol AHashMap", - "second Arc copy from string maps into symbol-id vectors" - ], - "publicDataHelpersChanged": false, - "rollingSemanticsChanged": false, - "adjustmentSemanticsChanged": false, - "pitSemanticsChanged": false, - "selectionResultsCached": false, - "netSourceLines": -10 - }, - "adjacentBaseline": { - "engineCommit": "cda249e9b4458770361b4cad570bd1e52f9eb67a", - "runs": [ - { - "runId": "btr_1788645721278_2764691_0", - "totalSeconds": 18.207, - "dataSeconds": 14.052, - "datasetConstructSeconds": 4.448 - }, - { - "runId": "btr_1788646439501_2764691_2", - "totalSeconds": 17.858, - "dataSeconds": 13.878, - "datasetConstructSeconds": 4.546 - } - ], - "datasetConstructMedianSeconds": 4.497 - }, - "primaryFiveYearContract": { - "runId": "btr_1788646914659_2773606_0", - "totalReturn": 0.9219861819172002, - "tradeCount": 26088, - "canonicalSha256": "b42fea66237d06eadb24f6b8c9e2760e7319fe3699f315b99e01f433ef2aa234", - "resultStoreSha256": "9ccf0c0fc6f5d72e974381ad4cd09a80241d7de99f2649f2b01736f7c80dc2c7", - "coldTotalSeconds": 16.322, - "coldDataSeconds": 12.815, - "datasetConstructSeconds": 3.899, - "hotTotalMedianSeconds": 3.402, - "hotEngineMedianSeconds": 2.792, - "terminalAuditStatus": "clean", - "resultConsistent": true - }, - "secondaryFiveYearContract": { - "runId": "btr_1788646986983_2773606_4", - "totalReturn": 1.1342962298106998, - "tradeCount": 19404, - "canonicalSha256": "0dbd3fad624097c673c4a5ec2f545f95e9c21d1d5337bb2b48ffeb22b16cb2b9", - "resultStoreSha256": "dd744dd6710bc28bb4ecd9ca912051292b9a1eefa0056e02c62a38cad85cf435", - "coldTotalSeconds": 15.292, - "coldDataSeconds": 12.238, - "datasetConstructSeconds": 3.895, - "terminalAuditStatus": "clean", - "resultConsistent": true - }, - "genericRankFiveYearContract": { - "runId": "btr_1788647058500_2773606_8", - "rankBy": "free_float_cap", - "totalReturn": 0.7140315244542004, - "tradeCount": 21876, - "canonicalSha256": "84367993911b42437939aad88f29379d007af6f5f171667ff7ad6677dd488fd2", - "resultStoreSha256": "865c00f81d17153285e391eb48ea51487e33fd3bba55a6c2a0ca280b07511077", - "coldTotalSeconds": 16.739, - "coldDataSeconds": 11.512, - "datasetConstructSeconds": 3.663, - "terminalAuditStatus": "clean", - "resultConsistent": true - }, - "minuteContract": { - "runId": "btr_1788647032685_2773606_6", - "totalReturn": 0.03473222656500008, - "tradeCount": 156, - "canonicalSha256": "457c086b1bca784fe83f447bb22451925e8f022456bb0fc7b54237a7a7886849", - "resultStoreSha256": "a77894eecf4e6a49168a38ff83cf27226e274b6850e10196892efa7d95be91cd", - "coldTotalSeconds": 0.814, - "hotTotalSeconds": 0.543, - "terminalAuditStatus": "clean", - "resultConsistent": true - }, - "performance": { - "candidateDatasetConstructSeconds": [3.899, 3.895, 3.663], - "candidateDatasetConstructMedianSeconds": 3.895, - "datasetConstructMedianImprovementPercent": 13.386702, - "primaryColdDataImprovementPercent": 7.659605, - "primaryColdTotalImprovementPercent": 8.601187, - "wallComparisonsIncludeHostLoadVariance": true - }, - "memory": { - "singleDatasetCandidateCurrentBytes": 11495055360, - "singleDatasetBaselineCurrentBytes": 11470495744, - "comparison": "within allocator and concurrent workload variance; no memory reduction claimed", - "repeatedManualCacheClearHighWaterExcluded": true - }, - "testGate": { - "corePassed": 538, - "ignoredManualBenchmarks": 8, - "failed": 0 - }, - "remoteArtifacts": [ - "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-primary-five-year-20260906.json", - "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-secondary-five-year-20260906.json", - "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-generic-rank-five-year-20260906.json", - "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-minute-20260906.json" - ], - "acceptance": { - "status": "accepted", - "reason": "three independent cold five-year builds reduce DataSet construction time, four distinct strategy contracts preserve exact canonical outputs, and duplicate string-keyed series indexes are removed rather than hidden behind another cache" - } -} diff --git a/docs/evidence/symbol-id-series-storage-rejection-20260906.json b/docs/evidence/symbol-id-series-storage-rejection-20260906.json new file mode 100644 index 0000000..8bc5943 --- /dev/null +++ b/docs/evidence/symbol-id-series-storage-rejection-20260906.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": "fidc-symbol-id-series-storage-rejection/v1", + "measuredAt": "2026-09-06T06:31:00+08:00", + "host": "192.168.31.177", + "candidateCommit": "5a7c49a4543b584503ff0d2c2ae513c68d25de8a", + "implementation": "remove duplicate string-keyed market and adjusted-close series maps and build symbol-id vectors directly", + "businessParity": { + "primaryFiveYearCanonicalSha256": "b42fea66237d06eadb24f6b8c9e2760e7319fe3699f315b99e01f433ef2aa234", + "secondaryFiveYearCanonicalSha256": "0dbd3fad624097c673c4a5ec2f545f95e9c21d1d5337bb2b48ffeb22b16cb2b9", + "genericRankFiveYearCanonicalSha256": "84367993911b42437939aad88f29379d007af6f5f171667ff7ad6677dd488fd2", + "minuteCanonicalSha256": "457c086b1bca784fe83f447bb22451925e8f022456bb0fc7b54237a7a7886849", + "allTerminalAuditsClean": true, + "allResultsConsistent": true + }, + "performance": { + "baselineDatasetConstructSeconds": [4.448, 4.546], + "candidateEarlyDatasetConstructSeconds": [3.899, 3.895, 3.663], + "candidateFinalRestartDatasetConstructSeconds": 4.472, + "candidateFinalRestartColdDataSeconds": 14.513, + "candidateFinalRestartColdTotalSeconds": 18.381, + "conclusion": "the apparent early improvement did not reproduce after a final restart under the current host phase; the final constructor time is equal to the adjacent baseline range" + }, + "memory": { + "baselineSingleDatasetCurrentBytes": 11470495744, + "candidateSingleDatasetCurrentBytes": 11473674240, + "conclusion": "no measurable resident-memory reduction" + }, + "testGate": { + "corePassed": 538, + "ignoredManualBenchmarks": 8, + "failed": 0 + }, + "remoteArtifacts": [ + "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-primary-five-year-20260906.json", + "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-secondary-five-year-20260906.json", + "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-generic-rank-five-year-20260906.json", + "/srv/fidc/canonical/run/fidc-private/evidence/symbol-id-series-minute-20260906.json", + "/srv/fidc/canonical/run/fidc-private/evidence/final-symbol-id-series-post-doc-sync-20260906.json" + ], + "decision": { + "status": "rejected_and_removed", + "reason": "the candidate preserved correctness but did not provide a stable end-to-end or memory improvement across restart validation; duplicate maps are not a proven material bottleneck", + "nextTarget": "remove the SourceRowRecord to DailySnapshot to DataSet multi-stage materialization, or publish a content-addressed base panel that can be mapped across restarts" + } +}