zsb/fidc-backtest-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(core): build sorted dataset components linearly

Browse Source
This commit is contained in:
boris
2026-08-28 07:11:19 +08:00
parent 3f67ee9134
commit c46dcf817b
1 changed files with 302 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/fidc-core/src/data.rs
+302 -6
View File
@@ -72,6 +72,12 @@ pub enum DataSetError {
},
#[error("benchmark snapshot missing for {date}")]
MissingBenchmark { date: NaiveDate },
#[error("{kind} snapshots are not strictly sorted by date and symbol at {date} / {symbol}")]
InvalidComponentOrder {
kind: &'static str,
date: NaiveDate,
symbol: String,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -1308,6 +1314,30 @@ impl DataSet {
)
}
pub fn from_date_symbol_sorted_components_with_actions_and_quotes(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
execution_quotes: Vec<IntradayExecutionQuote>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
Vec::new(),
Vec::new(),
Vec::new(),
true,
)
}
pub fn from_components_with_actions_quotes_and_futures(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
@@ -1389,6 +1419,35 @@ impl DataSet {
futures_params: Vec<FuturesTradingParameter>,
order_book_depth: Vec<IntradayOrderBookDepthLevel>,
factor_texts: Vec<FactorTextValue>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
futures_params,
order_book_depth,
factor_texts,
false,
)
}
#[allow(clippy::too_many_arguments)]
fn from_components_with_actions_quotes_futures_depth_and_factor_texts_ordered(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
execution_quotes: Vec<IntradayExecutionQuote>,
futures_params: Vec<FuturesTradingParameter>,
order_book_depth: Vec<IntradayOrderBookDepthLevel>,
factor_texts: Vec<FactorTextValue>,
date_symbol_sorted: bool,
) -> Result<Self, DataSetError> {
let benchmark_code = collect_benchmark_code(&benchmarks)?;
let calendar = TradingCalendar::new(benchmarks.iter().map(|item| item.date).collect());
@@ -1399,11 +1458,31 @@ impl DataSet {
.map(|instrument| (instrument.symbol.clone(), instrument))
.collect::<HashMap<_, _>>();
let mut market_by_date = group_by_date(market, |item| item.date);
sort_groups_by_symbol(&mut market_by_date, |item| item.symbol.as_str());
let market_by_date = if date_symbol_sorted {
group_date_symbol_sorted(
market,
|item| item.date,
|item| item.symbol.as_str(),
"market",
)?
} else {
let mut grouped = group_by_date(market, |item| item.date);
sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str());
grouped
};
let mut factor_by_date = group_by_date(factors, |item| item.date);
sort_groups_by_symbol(&mut factor_by_date, |item| item.symbol.as_str());
let factor_by_date = if date_symbol_sorted {
group_date_symbol_sorted(
factors,
|item| item.date,
|item| item.symbol.as_str(),
"factor",
)?
} else {
let mut grouped = group_by_date(factors, |item| item.date);
sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str());
grouped
};
let mut market_rows_by_symbol = AHashMap::<String, Vec<&DailyMarketSnapshot>>::new();
for row in market_by_date.values().flatten() {
if let Some(rows) = market_rows_by_symbol.get_mut(row.symbol.as_str()) {
@@ -1459,8 +1538,18 @@ impl DataSet {
.map(|item| ((item.date, item.symbol.clone(), item.field.clone()), item))
.collect::<HashMap<_, _>>();
let mut candidate_by_date = group_by_date(candidates, |item| item.date);
sort_groups_by_symbol(&mut candidate_by_date, |item| item.symbol.as_str());
let candidate_by_date = if date_symbol_sorted {
group_date_symbol_sorted(
candidates,
|item| item.date,
|item| item.symbol.as_str(),
"candidate",
)?
} else {
let mut grouped = group_by_date(candidates, |item| item.date);
sort_groups_by_symbol(&mut grouped, |item| item.symbol.as_str());
grouped
};
let symbol_id_by_code = build_symbol_id_index(
&instruments,
&market_by_date,
@@ -3477,6 +3566,58 @@ where
grouped
}
fn group_date_symbol_sorted<T, D, S>(
rows: Vec<T>,
date_of: D,
symbol_of: S,
kind: &'static str,
) -> Result<BTreeMap<NaiveDate, Vec<T>>, DataSetError>
where
D: Fn(&T) -> NaiveDate + Copy,
S: Fn(&T) -> &str + Copy,
{
let mut grouped = BTreeMap::<NaiveDate, Vec<T>>::new();
let mut rows = rows.into_iter().peekable();
while let Some(first) = rows.next() {
let date = date_of(&first);
if grouped
.last_key_value()
.is_some_and(|(previous, _)| *previous >= date)
{
return Err(DataSetError::InvalidComponentOrder {
kind,
date,
symbol: symbol_of(&first).to_string(),
});
}
let mut day_rows = vec![first];
while rows.peek().is_some_and(|row| date_of(row) == date) {
day_rows.push(rows.next().expect("peeked component row"));
}
if let Some(next) = rows.peek()
&& date_of(next) < date
{
return Err(DataSetError::InvalidComponentOrder {
kind,
date: date_of(next),
symbol: symbol_of(next).to_string(),
});
}
if let Some(window) = day_rows
.windows(2)
.find(|window| symbol_of(&window[0]) >= symbol_of(&window[1]))
{
return Err(DataSetError::InvalidComponentOrder {
kind,
date,
symbol: symbol_of(&window[1]).to_string(),
});
}
grouped.insert(date, day_rows);
}
Ok(grouped)
}
fn sort_groups_by_symbol<T, F>(groups: &mut BTreeMap<NaiveDate, Vec<T>>, symbol_of: F)
where
F: Fn(&T) -> &str + Copy,
@@ -3891,6 +4032,161 @@ mod tests {
}
}
fn instrument(symbol: &str) -> Instrument {
Instrument {
symbol: symbol.to_string(),
name: symbol.to_string(),
board: symbol
.rsplit_once('.')
.map(|(_, board)| board)
.unwrap_or("")
.to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
}
}
fn market_row_for(date: &str, symbol: &str, close: f64) -> DailyMarketSnapshot {
let mut row = market_row(date, close, 1_000_000);
row.symbol = symbol.to_string();
row
}
fn factor_row_for(date: &str, symbol: &str, market_cap_bn: f64) -> DailyFactorSnapshot {
DailyFactorSnapshot {
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
symbol: symbol.to_string(),
market_cap_bn,
free_float_cap_bn: market_cap_bn * 0.8,
pe_ttm: 0.0,
turnover_ratio: Some(0.02),
effective_turnover_ratio: Some(0.01),
extra_factors: NumericFactorMap::from([(Cow::Borrowed("quality"), 1.0)]),
}
}
fn candidate_row_for(date: &str, symbol: &str) -> CandidateEligibility {
CandidateEligibility {
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
symbol: symbol.to_string(),
is_st: false,
is_star_st: false,
is_new_listing: false,
is_paused: false,
allow_buy: true,
allow_sell: true,
is_kcb: false,
is_one_yuan: false,
risk_level_code: None,
}
}
#[test]
fn sorted_component_constructor_matches_generic_constructor() {
let instruments = vec![instrument("000001.SZ"), instrument("600000.SH")];
let market = vec![
market_row_for("2025-01-02", "000001.SZ", 10.0),
market_row_for("2025-01-02", "600000.SH", 12.0),
market_row_for("2025-01-03", "000001.SZ", 10.2),
market_row_for("2025-01-03", "600000.SH", 12.2),
];
let factors = vec![
factor_row_for("2025-01-02", "000001.SZ", 100.0),
factor_row_for("2025-01-02", "600000.SH", 120.0),
factor_row_for("2025-01-03", "000001.SZ", 102.0),
factor_row_for("2025-01-03", "600000.SH", 122.0),
];
let candidates = vec![
candidate_row_for("2025-01-02", "000001.SZ"),
candidate_row_for("2025-01-02", "600000.SH"),
candidate_row_for("2025-01-03", "000001.SZ"),
candidate_row_for("2025-01-03", "600000.SH"),
];
let benchmarks = vec![
benchmark_row("2025-01-02", 20.0),
benchmark_row("2025-01-03", 20.2),
];
let generic = DataSet::from_components(
instruments.clone(),
market.clone(),
factors.clone(),
candidates.clone(),
benchmarks.clone(),
)
.unwrap();
let sorted = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes(
instruments,
market,
factors,
candidates,
benchmarks,
Vec::new(),
Vec::new(),
)
.unwrap();
for date in ["2025-01-02", "2025-01-03"] {
let date = NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap();
for symbol in ["000001.SZ", "600000.SH"] {
assert_eq!(
generic.market(date, symbol).unwrap().close,
sorted.market(date, symbol).unwrap().close
);
assert_eq!(
generic.factor(date, symbol).unwrap().market_cap_bn,
sorted.factor(date, symbol).unwrap().market_cap_bn
);
assert_eq!(
generic.candidate(date, symbol).unwrap().allow_buy,
sorted.candidate(date, symbol).unwrap().allow_buy
);
}
}
}
#[test]
fn sorted_component_constructor_rejects_unsorted_or_duplicate_rows() {
let benchmarks = vec![
benchmark_row("2025-01-02", 20.0),
benchmark_row("2025-01-03", 20.2),
];
let unsorted = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes(
vec![instrument("000001.SZ")],
vec![
market_row_for("2025-01-03", "000001.SZ", 10.2),
market_row_for("2025-01-02", "000001.SZ", 10.0),
],
Vec::new(),
Vec::new(),
benchmarks.clone(),
Vec::new(),
Vec::new(),
);
assert!(matches!(
unsorted,
Err(DataSetError::InvalidComponentOrder { kind: "market", .. })
));
let duplicate = DataSet::from_date_symbol_sorted_components_with_actions_and_quotes(
vec![instrument("000001.SZ")],
vec![
market_row_for("2025-01-02", "000001.SZ", 10.0),
market_row_for("2025-01-02", "000001.SZ", 10.1),
],
Vec::new(),
Vec::new(),
benchmarks,
Vec::new(),
Vec::new(),
);
assert!(matches!(
duplicate,
Err(DataSetError::InvalidComponentOrder { kind: "market", .. })
));
}
#[test]
fn dataset_clone_shares_immutable_base_and_isolates_execution_quotes() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();