zsb/fidc-backtest-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(data): compact immutable snapshots with current execution rules

Browse Source
This commit is contained in:
boris
2026-09-12 16:02:24 +08:00
parent ffd23b9920
commit 828690a51e
22 changed files with 1430 additions and 1058 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/fidc-core/src/data.rs
+64 -63
View File
@@ -5,6 +5,7 @@ use std::sync::{Arc, OnceLock};
use ahash::AHashMap;
use chrono::{NaiveDate, NaiveDateTime};
use compact_str::CompactString;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use thiserror::Error;
@@ -122,7 +123,7 @@ pub enum PriceField {
pub struct DailyMarketSnapshot {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub symbol: CompactString,
pub timestamp: Option<String>,
pub day_open: f64,
pub open: f64,
@@ -199,13 +200,13 @@ impl DailyMarketSnapshot {
}
}
pub type NumericFactorMap = BTreeMap<Cow<'static, str>, f64>;
pub use crate::numeric_factors::NumericFactorMap;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DailyFactorSnapshot {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub symbol: CompactString,
pub market_cap_bn: f64,
pub free_float_cap_bn: f64,
pub pe_ttm: f64,
@@ -232,7 +233,7 @@ pub struct BenchmarkSnapshot {
pub struct CandidateEligibility {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub symbol: CompactString,
pub is_st: bool,
#[serde(default)]
pub is_star_st: bool,
@@ -1204,7 +1205,7 @@ impl SymbolPriceSeries {
fn snapshot_at(&self, index: usize) -> DailyMarketSnapshot {
DailyMarketSnapshot {
date: self.dates[index],
symbol: self.symbol.clone(),
symbol: self.symbol.as_str().into(),
timestamp: self.timestamps[index].clone(),
day_open: self.day_opens[index],
open: self.opens[index],
@@ -2865,7 +2866,7 @@ impl DataSet {
.filter_map(|snapshot| {
factor_numeric_value(snapshot, &field).map(|value| FactorValue {
date: snapshot.date,
symbol: snapshot.symbol.clone(),
symbol: snapshot.symbol.to_string(),
field: field.clone(),
value,
})
@@ -2959,7 +2960,7 @@ impl DataSet {
.or_else(|| row.extra_factors.get("margin_all"))
.is_some_and(|value| *value > 0.0)
})
.map(|row| row.symbol.clone())
.map(|row| row.symbol.to_string())
.collect::<Vec<_>>()
})
.unwrap_or_default();
@@ -3051,7 +3052,7 @@ impl DataSet {
if snapshot.prev_close.is_finite() && snapshot.prev_close > 0.0 {
Some(FactorValue {
date: snapshot.date,
symbol: snapshot.symbol.clone(),
symbol: snapshot.symbol.to_string(),
field: "price_change_rate".to_string(),
value: snapshot.close / snapshot.prev_close - 1.0,
})
@@ -3173,7 +3174,7 @@ impl DataSet {
if let Some(value) = factor_numeric_value(snapshot, field) {
return Some(FactorValue {
date: *factor_date,
symbol: snapshot.symbol.clone(),
symbol: snapshot.symbol.to_string(),
field: field.clone(),
value,
});
@@ -3524,7 +3525,7 @@ impl DataSet {
if let Some(value) = factor_numeric_value(snapshot, field) {
rows.push(FactorValue {
date: snapshot.date,
symbol: snapshot.symbol.clone(),
symbol: snapshot.symbol.to_string(),
field: output_field.clone(),
value,
});
@@ -4322,7 +4323,7 @@ fn daily_market_price_bar(snapshot: &DailyMarketSnapshot) -> PriceBar {
PriceBar {
date: snapshot.date,
timestamp: snapshot.timestamp.clone(),
symbol: snapshot.symbol.clone(),
symbol: snapshot.symbol.to_string(),
frequency: "1d".to_string(),
open: snapshot.open,
high: snapshot.high,
@@ -4383,7 +4384,7 @@ fn normalize_factor_snapshots(
{
return Err(DataSetError::ReservedTypedFactorInExtraMap {
date: snapshot.date,
symbol: snapshot.symbol,
symbol: snapshot.symbol.to_string(),
field: BACKWARD_ADJUSTMENT_FACTOR_FIELD,
});
}
@@ -4392,7 +4393,7 @@ fn normalize_factor_snapshots(
{
return Err(DataSetError::InvalidBackwardAdjustmentFactor {
date: snapshot.date,
symbol: snapshot.symbol,
symbol: snapshot.symbol.to_string(),
value,
});
}
@@ -4428,7 +4429,7 @@ fn normalize_factor_snapshots(
{
return Err(DataSetError::ReservedTypedFactorInExtraMap {
date: snapshot.date,
symbol: snapshot.symbol,
symbol: snapshot.symbol.to_string(),
field: BACKWARD_ADJUSTMENT_FACTOR_FIELD,
});
}
@@ -4510,21 +4511,21 @@ fn build_symbol_id_index(
for rows in market_by_date.values() {
for row in rows {
if !symbols.contains(row.symbol.as_str()) {
symbols.insert(row.symbol.clone());
symbols.insert(row.symbol.to_string());
}
}
}
for rows in factor_by_date.values() {
for row in rows {
if !symbols.contains(row.symbol.as_str()) {
symbols.insert(row.symbol.clone());
symbols.insert(row.symbol.to_string());
}
}
}
for rows in candidate_by_date.values() {
for row in rows {
if !symbols.contains(row.symbol.as_str()) {
symbols.insert(row.symbol.clone());
symbols.insert(row.symbol.to_string());
}
}
}
@@ -4942,7 +4943,7 @@ fn build_fundamental_universe_for_date(
continue;
}
rows.push(EligibleUniverseSnapshot {
symbol: factor.symbol.clone(),
symbol: factor.symbol.to_string(),
market_cap_bn,
free_float_cap_bn: decision_free_float_cap_bn(factor),
});
@@ -5012,7 +5013,7 @@ fn build_eligible_universe_for_date_from_factors(
date,
candidate,
market,
instruments.get(&factor.symbol),
instruments.get(factor.symbol.as_str()),
risk_config,
)
.is_some()
@@ -5025,7 +5026,7 @@ fn build_eligible_universe_for_date_from_factors(
}
let free_float_cap_bn = decision_free_float_cap_bn(factor);
rows.push(EligibleUniverseSnapshot {
symbol: factor.symbol.clone(),
symbol: factor.symbol.to_string(),
market_cap_bn,
free_float_cap_bn,
});
@@ -5042,7 +5043,7 @@ fn build_eligible_universe_for_date_from_factors(
pub(crate) fn missing_candidate_risk_state(date: NaiveDate, symbol: &str) -> CandidateEligibility {
CandidateEligibility {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
is_st: false,
is_star_st: false,
is_new_listing: false,
@@ -5070,7 +5071,7 @@ mod tests {
fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot {
DailyMarketSnapshot {
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
timestamp: None,
day_open: prev_close,
open: prev_close,
@@ -5352,10 +5353,10 @@ mod tests {
for (symbol_index, symbol) in symbols.into_iter().enumerate().rev() {
let close = 10.0 + date_index as f64 + symbol_index as f64;
let mut market_row = market_row(&date_text, close, 1_000_000);
market_row.symbol = symbol.to_string();
market_row.symbol = symbol.into();
let factor_row = DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
market_cap_bn: 100.0 + close,
free_float_cap_bn: 80.0 + close,
pe_ttm: 0.0,
@@ -5366,7 +5367,7 @@ mod tests {
};
let candidate_row = CandidateEligibility {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
is_st: false,
is_star_st: false,
is_new_listing: false,
@@ -5534,12 +5535,12 @@ mod tests {
};
let market = |symbol: &str, close: f64| {
let mut row = market_row("2025-01-02", close, 1_000_000);
row.symbol = symbol.to_string();
row.symbol = symbol.into();
row
};
let factor = |symbol: &str, market_cap_bn: f64| DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
market_cap_bn,
free_float_cap_bn: market_cap_bn,
pe_ttm: 0.0,
@@ -5550,7 +5551,7 @@ mod tests {
};
let candidate = |symbol: &str| CandidateEligibility {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
is_st: false,
is_star_st: false,
is_new_listing: false,
@@ -5773,7 +5774,7 @@ mod tests {
.iter()
.map(|date| DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
@@ -5977,7 +5978,7 @@ mod tests {
.enumerate()
.map(|(index, symbol)| {
let mut row = market_row("2025-01-02", 10.0 + index as f64 / 1000.0, 1_000_000);
row.symbol = symbol.clone();
row.symbol = symbol.clone().into();
row
})
.collect::<Vec<_>>();
@@ -5986,7 +5987,7 @@ mod tests {
.enumerate()
.map(|(index, symbol)| DailyFactorSnapshot {
date,
symbol: symbol.clone(),
symbol: symbol.clone().into(),
market_cap_bn: 10.0 + index as f64 / 1000.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
@@ -6000,7 +6001,7 @@ mod tests {
.iter()
.map(|symbol| CandidateEligibility {
date,
symbol: symbol.clone(),
symbol: symbol.clone().into(),
is_st: false,
is_star_st: false,
is_new_listing: false,
@@ -6289,14 +6290,14 @@ mod tests {
fn factor_numeric_value_normalizes_fields_without_changing_aliases() {
let snapshot = DailyFactorSnapshot {
date: NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap(),
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 12.5,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: Some(1.25),
extra_factors: BTreeMap::from([("custom_factor".into(), 3.5)]),
extra_factors: From::from([("custom_factor".into(), 3.5)]),
};
assert_eq!(factor_numeric_value(&snapshot, " MARKET_CAP "), Some(12.5));
@@ -6312,14 +6313,14 @@ mod tests {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let clean = normalize_factor_snapshots(vec![DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 1.0,
free_float_cap_bn: 1.0,
pe_ttm: 1.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: None,
extra_factors: BTreeMap::from([(Cow::Borrowed("amount"), 10.0)]),
extra_factors: From::from([(Cow::Borrowed("amount"), 10.0)]),
}])
.expect("normalize clean factor snapshot");
assert!(matches!(
@@ -6329,14 +6330,14 @@ mod tests {
let dirty = normalize_factor_snapshots(vec![DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 1.0,
free_float_cap_bn: 1.0,
pe_ttm: 1.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: None,
extra_factors: BTreeMap::from([
extra_factors: From::from([
(Cow::Owned(" CUSTOM_FACTOR ".to_string()), 2.0),
(Cow::Borrowed("bad_nan"), f64::NAN),
]),
@@ -6351,7 +6352,7 @@ mod tests {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let snapshot = |adjustment_factor_backward1, extra_factors| DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 1.0,
free_float_cap_bn: 1.0,
pe_ttm: 1.0,
@@ -6364,12 +6365,12 @@ mod tests {
assert!(matches!(
normalize_factor_snapshots(vec![snapshot(
Some(1.0),
BTreeMap::from([(Cow::Borrowed(BACKWARD_ADJUSTMENT_FACTOR_FIELD), 1.0)]),
BTreeMap::from([(Cow::Borrowed(BACKWARD_ADJUSTMENT_FACTOR_FIELD), 1.0)]).into(),
)]),
Err(DataSetError::ReservedTypedFactorInExtraMap { .. })
));
assert!(matches!(
normalize_factor_snapshots(vec![snapshot(Some(0.0), BTreeMap::new())]),
normalize_factor_snapshots(vec![snapshot(Some(0.0), BTreeMap::new().into())]),
Err(DataSetError::InvalidBackwardAdjustmentFactor { .. })
));
for field in [
@@ -6382,7 +6383,7 @@ mod tests {
matches!(
normalize_factor_snapshots(vec![snapshot(
typed_value,
BTreeMap::from([(Cow::Borrowed(field), 2.0)]),
BTreeMap::from([(Cow::Borrowed(field), 2.0)]).into(),
)]),
Err(DataSetError::ReservedTypedFactorInExtraMap { .. })
),
@@ -6478,14 +6479,14 @@ mod tests {
}
DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: None,
extra_factors,
extra_factors: extra_factors.into(),
}
})
.collect(),
@@ -6537,14 +6538,14 @@ mod tests {
.iter()
.map(|date| DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: Some(1.0),
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
})
.collect(),
Vec::new(),
@@ -6831,14 +6832,14 @@ mod tests {
.zip(factors)
.map(|(date, factor)| DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: Some(factor),
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
})
.collect(),
Vec::new(),
@@ -6929,14 +6930,14 @@ mod tests {
.iter()
.map(|date| DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
adjustment_factor_backward1: (*date != dates[3]).then_some(1.0),
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
})
.collect(),
Vec::new(),
@@ -7008,7 +7009,7 @@ mod tests {
};
let market = |symbol: &str, prev_close: f64, close: f64| DailyMarketSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
timestamp: Some("2025-01-06 10:18:00".to_string()),
day_open: prev_close,
open: prev_close,
@@ -7032,18 +7033,18 @@ mod tests {
let factor =
|symbol: &str, market_cap_bn: f64, free_float_cap_bn: f64| DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
market_cap_bn,
free_float_cap_bn,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
adjustment_factor_backward1: None,
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
};
let candidate = |symbol: &str| CandidateEligibility {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
is_st: false,
is_star_st: false,
is_new_listing: false,
@@ -7108,7 +7109,7 @@ mod tests {
}],
vec![DailyMarketSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
timestamp: Some("2025-01-06 10:18:00".to_string()),
day_open: 10.0,
open: 10.0,
@@ -7131,14 +7132,14 @@ mod tests {
}],
vec![DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
market_cap_bn: 10.0,
free_float_cap_bn: 9.0,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
adjustment_factor_backward1: None,
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
}],
Vec::new(),
vec![BenchmarkSnapshot {
@@ -7193,7 +7194,7 @@ mod tests {
}],
vec![DailyMarketSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
timestamp: Some("2025-01-06 10:18:00".to_string()),
day_open: 10.0,
open: 10.0,
@@ -7216,18 +7217,18 @@ mod tests {
}],
vec![DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
market_cap_bn: 10.0,
free_float_cap_bn: 9.0,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
adjustment_factor_backward1: None,
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
}],
vec![CandidateEligibility {
date,
symbol: symbol.to_string(),
symbol: symbol.into(),
is_st: false,
is_star_st: false,
is_new_listing: false,
@@ -7268,14 +7269,14 @@ mod tests {
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
let factor = DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
symbol: "000001.SZ".into(),
market_cap_bn: 12.0,
free_float_cap_bn: 4.0,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
adjustment_factor_backward1: None,
extra_factors: BTreeMap::new(),
extra_factors: Default::default(),
};
assert!((decision_market_cap_bn(&factor) - 12.0).abs() < 1e-9);