zsb/fidc-backtest-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
732c3cfbf57500925e1dc7b2b876e58473f3fa9c
fidc-backtest-engine/crates/fidc-core/src/data.rs
T

5211 lines
174 KiB
Rust
Raw Blame History

use std::borrow::Cow;
use std::cmp::Reverse;
use std::collections::{BTreeMap, BTreeSet, BinaryHeap, HashMap, HashSet};
use std::sync::{Arc, OnceLock};
use ahash::AHashMap;
use chrono::{NaiveDate, NaiveDateTime};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::calendar::TradingCalendar;
use crate::futures::FuturesTradingParameter;
use crate::instrument::Instrument;
use crate::risk_control::{ChinaAShareRiskControl, FidcRiskControlConfig};
const BACKWARD_ADJUSTMENT_FACTOR_FIELD: &str = "adjustment_factor_backward1";
mod date_format {
use chrono::NaiveDate;
use serde::{self, Deserialize, Deserializer, Serializer};
const FORMAT: &str = "%Y-%m-%d";
pub fn serialize<S>(date: &NaiveDate, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&date.format(FORMAT).to_string())
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<NaiveDate, D::Error>
where
D: Deserializer<'de>,
{
let text = String::deserialize(deserializer)?;
NaiveDate::parse_from_str(&text, FORMAT).map_err(serde::de::Error::custom)
}
}
mod datetime_format {
use chrono::NaiveDateTime;
use serde::{self, Deserialize, Deserializer, Serializer};
const FORMAT: &str = "%Y-%m-%d %H:%M:%S";
pub fn serialize<S>(date: &NaiveDateTime, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&date.format(FORMAT).to_string())
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<NaiveDateTime, D::Error>
where
D: Deserializer<'de>,
{
let text = String::deserialize(deserializer)?;
NaiveDateTime::parse_from_str(&text, FORMAT).map_err(serde::de::Error::custom)
}
}
#[derive(Debug, Error)]
pub enum DataSetError {
#[error("benchmark file contains multiple benchmark codes")]
MultipleBenchmarks,
#[error("missing data for {kind} on {date} / {symbol}")]
MissingSnapshot {
kind: &'static str,
date: NaiveDate,
symbol: String,
},
#[error("benchmark snapshot missing for {date}")]
MissingBenchmark { date: NaiveDate },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PriceField {
DayOpen,
Open,
Close,
Last,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DailyMarketSnapshot {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub timestamp: Option<String>,
pub day_open: f64,
pub open: f64,
pub high: f64,
pub low: f64,
pub close: f64,
pub last_price: f64,
pub bid1: f64,
pub ask1: f64,
pub prev_close: f64,
pub volume: u64,
pub minute_volume: u64,
pub bid1_volume: u64,
pub ask1_volume: u64,
pub trading_phase: Option<String>,
pub paused: bool,
pub upper_limit: f64,
pub lower_limit: f64,
pub price_tick: f64,
}
impl DailyMarketSnapshot {
pub fn price(&self, field: PriceField) -> f64 {
match field {
PriceField::DayOpen => self.day_open,
PriceField::Open => self.open,
PriceField::Close => self.close,
PriceField::Last => self.last_price,
}
}
pub fn buy_price(&self, field: PriceField) -> f64 {
match field {
PriceField::Last if self.ask1.is_finite() && self.ask1 > 0.0 => self.ask1,
_ => self.price(field),
}
}
pub fn sell_price(&self, field: PriceField) -> f64 {
match field {
PriceField::Last if self.bid1.is_finite() && self.bid1 > 0.0 => self.bid1,
_ => self.price(field),
}
}
pub fn liquidity_for_buy(&self) -> u64 {
self.ask1_volume
}
pub fn liquidity_for_sell(&self) -> u64 {
self.bid1_volume
}
pub fn effective_price_tick(&self) -> f64 {
if self.price_tick.is_finite() && self.price_tick > 0.0 {
self.price_tick
} else {
0.01
}
}
pub fn is_at_upper_limit_price(&self, price: f64) -> bool {
if !self.upper_limit.is_finite() || self.upper_limit <= 0.0 {
return false;
}
price >= self.upper_limit - 1e-9
}
pub fn is_at_lower_limit_price(&self, price: f64) -> bool {
if !self.lower_limit.is_finite() || self.lower_limit <= 0.0 {
return false;
}
price <= self.lower_limit + 1e-9
}
}
pub type NumericFactorMap = BTreeMap<Cow<'static, str>, f64>;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DailyFactorSnapshot {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub market_cap_bn: f64,
pub free_float_cap_bn: f64,
pub pe_ttm: f64,
pub turnover_ratio: Option<f64>,
pub effective_turnover_ratio: Option<f64>,
#[serde(default)]
pub extra_factors: NumericFactorMap,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenchmarkSnapshot {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub benchmark: String,
pub open: f64,
pub close: f64,
pub prev_close: f64,
pub volume: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CandidateEligibility {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub is_st: bool,
#[serde(default)]
pub is_star_st: bool,
pub is_new_listing: bool,
pub is_paused: bool,
pub allow_buy: bool,
pub allow_sell: bool,
pub is_kcb: bool,
pub is_one_yuan: bool,
#[serde(default)]
pub risk_level_code: Option<String>,
}
impl CandidateEligibility {
pub fn eligible_for_selection(&self) -> bool {
!self.is_st
&& !self.is_star_st
&& !self.is_new_listing
&& !self.is_paused
&& !self.is_kcb
&& !self.is_one_yuan
&& self.allow_buy
&& self.allow_sell
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CorporateAction {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
#[serde(default, with = "optional_date_format")]
pub payable_date: Option<NaiveDate>,
pub share_cash: f64,
pub share_bonus: f64,
pub share_gift: f64,
pub issue_quantity: f64,
pub issue_price: f64,
pub reform: bool,
pub adjust_factor: Option<f64>,
#[serde(default)]
pub successor_symbol: Option<String>,
#[serde(default)]
pub successor_ratio: Option<f64>,
#[serde(default)]
pub successor_cash: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntradayExecutionQuote {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
#[serde(with = "datetime_format")]
pub timestamp: NaiveDateTime,
pub last_price: f64,
pub bid1: f64,
pub ask1: f64,
pub bid1_volume: u64,
pub ask1_volume: u64,
#[serde(default)]
pub volume_delta: u64,
#[serde(default)]
pub amount_delta: f64,
pub trading_phase: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IntradayOrderBookDepthLevel {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
#[serde(with = "datetime_format")]
pub timestamp: NaiveDateTime,
pub level: u8,
pub bid_price: f64,
pub bid_volume: u64,
pub ask_price: f64,
pub ask_volume: u64,
}
impl IntradayOrderBookDepthLevel {
pub fn executable_price(&self, side: crate::events::OrderSide) -> Option<f64> {
match side {
crate::events::OrderSide::Buy if self.ask_price.is_finite() && self.ask_price > 0.0 => {
Some(self.ask_price)
}
crate::events::OrderSide::Sell
if self.bid_price.is_finite() && self.bid_price > 0.0 =>
{
Some(self.bid_price)
}
_ => None,
}
}
pub fn executable_volume(&self, side: crate::events::OrderSide) -> u64 {
match side {
crate::events::OrderSide::Buy => self.ask_volume,
crate::events::OrderSide::Sell => self.bid_volume,
}
}
}
impl IntradayExecutionQuote {
pub fn buy_price(&self) -> Option<f64> {
if self.ask1.is_finite() && self.ask1 > 0.0 {
Some(self.ask1)
} else if self.last_price.is_finite() && self.last_price > 0.0 {
Some(self.last_price)
} else {
None
}
}
pub fn sell_price(&self) -> Option<f64> {
if self.bid1.is_finite() && self.bid1 > 0.0 {
Some(self.bid1)
} else if self.last_price.is_finite() && self.last_price > 0.0 {
Some(self.last_price)
} else {
None
}
}
}
/// A borrowed, timestamp-ordered merge of the execution-quote streams for one
/// trading day. The iterator keeps only stream cursors and never clones quote
/// payloads; callers decide how much of the day they need to retain.
pub struct ExecutionQuoteIterator<'a> {
streams: Vec<(&'a str, &'a [IntradayExecutionQuote])>,
heap: BinaryHeap<Reverse<(NaiveDateTime, usize, usize)>>,
}
impl<'a> ExecutionQuoteIterator<'a> {
fn new(
rows_by_symbol: Option<&'a HashMap<String, Vec<IntradayExecutionQuote>>>,
symbols: Option<&BTreeSet<String>>,
) -> Self {
let mut streams = rows_by_symbol
.into_iter()
.flat_map(|rows_by_symbol| rows_by_symbol.iter())
.filter(|(symbol, _)| {
symbols
.map(|allowed_symbols| allowed_symbols.contains(*symbol))
.unwrap_or(true)
})
.map(|(symbol, rows)| (symbol.as_str(), rows.as_slice()))
.collect::<Vec<_>>();
streams.sort_by_key(|(symbol, _)| *symbol);
let mut heap = BinaryHeap::with_capacity(streams.len());
for (stream_index, (_, rows)) in streams.iter().enumerate() {
if let Some(first) = rows.first() {
heap.push(Reverse((first.timestamp, stream_index, 0)));
}
}
Self { streams, heap }
}
}
impl<'a> Iterator for ExecutionQuoteIterator<'a> {
type Item = &'a IntradayExecutionQuote;
fn next(&mut self) -> Option<Self::Item> {
let Reverse((_timestamp, stream_index, row_index)) = self.heap.pop()?;
let rows = self.streams.get(stream_index)?.1;
let quote = rows.get(row_index)?;
let next_index = row_index + 1;
if let Some(next) = rows.get(next_index) {
self.heap
.push(Reverse((next.timestamp, stream_index, next_index)));
}
Some(quote)
}
}
impl CorporateAction {
pub fn split_ratio(&self) -> f64 {
1.0 + self.share_bonus.max(0.0) + self.share_gift.max(0.0)
}
pub fn has_effect(&self) -> bool {
self.share_cash.abs() > f64::EPSILON
|| (self.split_ratio() - 1.0).abs() > f64::EPSILON
|| self.issue_quantity.abs() > f64::EPSILON
|| self.reform
|| self.has_successor_conversion()
}
pub fn has_successor_conversion(&self) -> bool {
self.successor_symbol
.as_ref()
.is_some_and(|symbol| !symbol.trim().is_empty())
&& self.successor_ratio_value() > 0.0
}
pub fn successor_ratio_value(&self) -> f64 {
self.successor_ratio
.filter(|ratio| ratio.is_finite() && *ratio > 0.0)
.unwrap_or(1.0)
}
pub fn successor_cash_value(&self) -> f64 {
self.successor_cash
.filter(|cash| cash.is_finite())
.unwrap_or(0.0)
}
}
#[derive(Debug, Clone)]
pub struct DailySnapshotBundle {
pub date: NaiveDate,
pub benchmark: BenchmarkSnapshot,
pub market: Vec<DailyMarketSnapshot>,
pub factors: Vec<DailyFactorSnapshot>,
pub candidates: Vec<CandidateEligibility>,
pub corporate_actions: Vec<CorporateAction>,
}
#[derive(Debug, Clone)]
pub struct DataSetSnapshotComponents {
pub instruments: Vec<Instrument>,
pub market: Vec<DailyMarketSnapshot>,
pub factors: Vec<DailyFactorSnapshot>,
pub candidates: Vec<CandidateEligibility>,
pub benchmarks: Vec<BenchmarkSnapshot>,
pub corporate_actions: Vec<CorporateAction>,
pub execution_quotes: Vec<IntradayExecutionQuote>,
}
#[derive(Debug, Clone, Serialize)]
pub struct PriceBar {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub timestamp: Option<String>,
pub symbol: String,
pub frequency: String,
pub open: f64,
pub high: f64,
pub low: f64,
pub close: f64,
pub last_price: f64,
pub volume: u64,
pub amount: f64,
pub bid1: f64,
pub ask1: f64,
pub bid1_volume: u64,
pub ask1_volume: u64,
}
#[derive(Debug, Clone, Serialize)]
pub struct DividendRecord {
#[serde(with = "date_format")]
pub ex_dividend_date: NaiveDate,
#[serde(with = "date_format")]
pub payable_date: NaiveDate,
pub symbol: String,
pub dividend_cash_before_tax: f64,
pub round_lot: u32,
}
#[derive(Debug, Clone, Serialize)]
pub struct SplitRecord {
#[serde(with = "date_format")]
pub ex_dividend_date: NaiveDate,
pub symbol: String,
pub split_ratio: f64,
}
#[derive(Debug, Clone, Serialize)]
pub struct FactorValue {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub field: String,
pub value: f64,
}
#[derive(Debug, Clone, Serialize)]
pub struct FactorTextValue {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub field: String,
pub value: String,
}
#[derive(Debug, Clone, Serialize)]
pub struct SecuritiesMarginRecord {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub symbol: String,
pub field: String,
pub value: f64,
}
#[derive(Debug, Clone, Serialize)]
pub struct YieldCurvePoint {
#[serde(with = "date_format")]
pub date: NaiveDate,
pub tenor: String,
pub value: f64,
}
#[derive(Debug, Clone)]
pub struct EligibleUniverseSnapshot {
pub symbol: String,
pub market_cap_bn: f64,
pub free_float_cap_bn: f64,
}
pub fn decision_market_cap_bn(factor: &DailyFactorSnapshot) -> f64 {
factor.market_cap_bn
}
pub fn decision_free_float_cap_bn(factor: &DailyFactorSnapshot) -> f64 {
factor.free_float_cap_bn
}
#[derive(Debug, Clone)]
struct SymbolPriceSeries {
symbol: String,
dates: Vec<NaiveDate>,
timestamps: Vec<Option<String>>,
day_opens: Vec<f64>,
opens: Vec<f64>,
highs: Vec<f64>,
lows: Vec<f64>,
closes: Vec<f64>,
prev_closes: Vec<f64>,
last_prices: Vec<f64>,
bid1s: Vec<f64>,
ask1s: Vec<f64>,
volumes: Vec<u64>,
minute_volumes: Vec<u64>,
bid1_volumes: Vec<u64>,
ask1_volumes: Vec<u64>,
trading_phases: Vec<Option<String>>,
paused: Vec<bool>,
upper_limits: Vec<f64>,
lower_limits: Vec<f64>,
price_ticks: Vec<f64>,
open_prefix: Vec<f64>,
close_prefix: Vec<f64>,
prev_close_prefix: Vec<f64>,
last_prefix: Vec<f64>,
valid_volume_sum_prefix: Vec<f64>,
valid_volume_count_prefix: Vec<usize>,
valid_volume_start_by_count: Vec<usize>,
}
#[derive(Debug, Clone)]
struct AdjustedCloseSeries {
dates: Vec<NaiveDate>,
backward_factors: Vec<Option<f64>>,
back_adjusted_closes: Vec<Option<f64>>,
back_adjusted_close_prefix: Vec<f64>,
missing_back_adjusted_close_prefix: Vec<u32>,
}
impl AdjustedCloseSeries {
fn new(market: &SymbolPriceSeries, factor_rows: &[&DailyFactorSnapshot]) -> Option<Self> {
debug_assert!(
factor_rows
.windows(2)
.all(|window| window[0].date <= window[1].date)
);
let mut backward_factors = Vec::with_capacity(market.dates.len());
let mut back_adjusted_closes = Vec::with_capacity(market.dates.len());
let mut back_adjusted_close_prefix = Vec::with_capacity(market.dates.len() + 1);
let mut missing_back_adjusted_close_prefix = Vec::with_capacity(market.dates.len() + 1);
back_adjusted_close_prefix.push(0.0);
missing_back_adjusted_close_prefix.push(0);
let mut factor_index = 0usize;
for (date, close) in market.dates.iter().zip(&market.closes) {
while factor_rows
.get(factor_index)
.is_some_and(|snapshot| snapshot.date < *date)
{
factor_index += 1;
}
let factor = factor_rows
.get(factor_index)
.filter(|snapshot| snapshot.date == *date)
.and_then(|snapshot| {
snapshot
.extra_factors
.get(BACKWARD_ADJUSTMENT_FACTOR_FIELD)
.copied()
})
.filter(|factor| factor.is_finite() && *factor > 0.0);
let back_adjusted_close = factor
.filter(|_| close.is_finite() && *close > 0.0)
.map(|factor| close * factor);
backward_factors.push(factor);
back_adjusted_closes.push(back_adjusted_close);
back_adjusted_close_prefix.push(
back_adjusted_close_prefix
.last()
.copied()
.unwrap_or_default()
+ back_adjusted_close.unwrap_or_default(),
);
missing_back_adjusted_close_prefix.push(
missing_back_adjusted_close_prefix
.last()
.copied()
.unwrap_or_default()
+ u32::from(back_adjusted_close.is_none()),
);
}
Some(Self {
dates: market.dates.clone(),
backward_factors,
back_adjusted_closes,
back_adjusted_close_prefix,
missing_back_adjusted_close_prefix,
})
}
fn current_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = match self.dates.binary_search(&date) {
Ok(index) => index + 1,
Err(0) => return None,
Err(index) => index,
};
if end < lookback {
return None;
}
let base_factor = self.backward_factors.get(end - 1).copied().flatten()?;
let start = end - lookback;
if self.missing_back_adjusted_close_prefix[end]
!= self.missing_back_adjusted_close_prefix[start]
{
return None;
}
let sum = self.back_adjusted_close_prefix[end] - self.back_adjusted_close_prefix[start];
if !sum.is_finite() {
return None;
}
Some(normalize_rolling_factor(
sum / lookback as f64 / base_factor,
12,
))
}
fn decision_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
let end = self.decision_end_index(date)?;
self.decision_moving_average_at_end(end, lookback)
}
fn decision_end_index(&self, date: NaiveDate) -> Option<usize> {
match self.dates.binary_search(&date) {
Ok(index) => Some(index),
Err(0) => None,
Err(index) => Some(index),
}
}
fn decision_moving_average_at_end(&self, end: usize, lookback: usize) -> Option<f64> {
if lookback == 0 {
return None;
}
if end < lookback {
return None;
}
let base_factor = self.backward_factors.get(end - 1).copied().flatten()?;
let start = end - lookback;
if self.missing_back_adjusted_close_prefix[end]
!= self.missing_back_adjusted_close_prefix[start]
{
return None;
}
let sum = self.back_adjusted_close_prefix[end] - self.back_adjusted_close_prefix[start];
if !sum.is_finite() {
return None;
}
Some(normalize_rolling_factor(
sum / lookback as f64 / base_factor,
12,
))
}
fn values(&self, date: NaiveDate, lookback: usize, include_now: bool) -> Vec<f64> {
if lookback == 0 {
return Vec::new();
}
let end = match self.dates.binary_search(&date) {
Ok(index) => index + usize::from(include_now),
Err(0) => return Vec::new(),
Err(index) => index,
};
if end == 0 {
return Vec::new();
}
let start = end.saturating_sub(lookback);
let Some(base_factor) = self.backward_factors.get(end - 1).copied().flatten() else {
return Vec::new();
};
self.back_adjusted_closes[start..end]
.iter()
.copied()
.collect::<Option<Vec<_>>>()
.map(|values| {
values
.into_iter()
.map(|value| normalize_rolling_factor(value / base_factor, 12))
.collect()
})
.unwrap_or_default()
}
fn latest_back_adjusted_close(&self, date: NaiveDate) -> Option<f64> {
let index = match self.dates.binary_search(&date) {
Ok(index) => index,
Err(0) => return None,
Err(index) => index - 1,
};
self.back_adjusted_closes
.get(index)
.copied()
.flatten()
.filter(|value| value.is_finite() && *value > 0.0)
}
}
impl SymbolPriceSeries {
#[cfg(test)]
fn new<'a, I>(symbol: String, rows: I) -> Self
where
I: IntoIterator<Item = &'a DailyMarketSnapshot>,
{
let mut sorted = rows.into_iter().collect::<Vec<_>>();
sorted.sort_by_key(|row| row.date);
Self::from_sorted_rows(symbol, sorted)
}
fn from_sorted_rows(symbol: String, rows: Vec<&DailyMarketSnapshot>) -> Self {
debug_assert!(
rows.windows(2)
.all(|window| window[0].date <= window[1].date)
);
let row_count = rows.len();
let mut dates = Vec::with_capacity(row_count);
let mut timestamps = Vec::with_capacity(row_count);
let mut day_opens = Vec::with_capacity(row_count);
let mut opens = Vec::with_capacity(row_count);
let mut highs = Vec::with_capacity(row_count);
let mut lows = Vec::with_capacity(row_count);
let mut closes = Vec::with_capacity(row_count);
let mut prev_closes = Vec::with_capacity(row_count);
let mut last_prices = Vec::with_capacity(row_count);
let mut bid1s = Vec::with_capacity(row_count);
let mut ask1s = Vec::with_capacity(row_count);
let mut volumes = Vec::with_capacity(row_count);
let mut minute_volumes = Vec::with_capacity(row_count);
let mut bid1_volumes = Vec::with_capacity(row_count);
let mut ask1_volumes = Vec::with_capacity(row_count);
let mut trading_phases = Vec::with_capacity(row_count);
let mut paused = Vec::with_capacity(row_count);
let mut upper_limits = Vec::with_capacity(row_count);
let mut lower_limits = Vec::with_capacity(row_count);
let mut price_ticks = Vec::with_capacity(row_count);
for row in rows {
dates.push(row.date);
timestamps.push(row.timestamp.clone());
day_opens.push(row.day_open);
opens.push(row.open);
highs.push(row.high);
lows.push(row.low);
closes.push(row.close);
prev_closes.push(row.prev_close);
last_prices.push(row.last_price);
bid1s.push(row.bid1);
ask1s.push(row.ask1);
volumes.push(row.volume);
minute_volumes.push(row.minute_volume);
bid1_volumes.push(row.bid1_volume);
ask1_volumes.push(row.ask1_volume);
trading_phases.push(row.trading_phase.clone());
paused.push(row.paused);
upper_limits.push(row.upper_limit);
lower_limits.push(row.lower_limit);
price_ticks.push(row.price_tick);
}
let open_prefix = prefix_sums(&opens);
let close_prefix = prefix_sums(&closes);
let prev_close_prefix = prefix_sums(&prev_closes);
let last_prefix = prefix_sums(&last_prices);
let mut valid_volume_sum_prefix = Vec::with_capacity(volumes.len() + 1);
let mut valid_volume_count_prefix = Vec::with_capacity(volumes.len() + 1);
valid_volume_sum_prefix.push(0.0);
valid_volume_count_prefix.push(0);
for volume in &volumes {
let valid = *volume > 0;
valid_volume_sum_prefix.push(
valid_volume_sum_prefix.last().copied().unwrap_or_default()
+ if valid { *volume as f64 } else { 0.0 },
);
valid_volume_count_prefix.push(
valid_volume_count_prefix
.last()
.copied()
.unwrap_or_default()
+ usize::from(valid),
);
}
let valid_volume_count = valid_volume_count_prefix
.last()
.copied()
.unwrap_or_default();
let mut valid_volume_start_by_count = vec![0usize; valid_volume_count + 1];
for (index, count) in valid_volume_count_prefix.iter().copied().enumerate() {
valid_volume_start_by_count[count] = index;
}
Self {
symbol,
dates,
timestamps,
day_opens,
opens,
highs,
lows,
closes,
prev_closes,
last_prices,
bid1s,
ask1s,
volumes,
minute_volumes,
bid1_volumes,
ask1_volumes,
trading_phases,
paused,
upper_limits,
lower_limits,
price_ticks,
open_prefix,
close_prefix,
prev_close_prefix,
last_prefix,
valid_volume_sum_prefix,
valid_volume_count_prefix,
valid_volume_start_by_count,
}
}
fn moving_average(&self, date: NaiveDate, lookback: usize, field: PriceField) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = self.end_index(date)?;
if end < lookback {
return None;
}
let start = end - lookback;
let prefix = self.prefix_for(field);
let sum = prefix[end] - prefix[start];
Some(sum / lookback as f64)
}
fn trailing_values(&self, date: NaiveDate, lookback: usize, field: PriceField) -> Vec<f64> {
let Some(end) = self.end_index(date) else {
return Vec::new();
};
let start = end.saturating_sub(lookback);
self.price_values_for(field)[start..end].to_vec()
}
fn trailing_snapshots(
&self,
date: NaiveDate,
lookback: usize,
include_now: bool,
) -> Vec<DailyMarketSnapshot> {
if lookback == 0 {
return Vec::new();
}
let end = if include_now {
self.end_index(date)
} else {
self.previous_completed_end_index(date)
};
let Some(end) = end else {
return Vec::new();
};
let start = end.saturating_sub(lookback);
(start..end).map(|index| self.snapshot_at(index)).collect()
}
fn trailing_numeric_values(
&self,
date: NaiveDate,
lookback: usize,
field: &str,
include_now: bool,
) -> Vec<f64> {
if lookback == 0 {
return Vec::new();
}
let end = if include_now {
self.end_index(date)
} else {
self.previous_completed_end_index(date)
};
let Some(end) = end else {
return Vec::new();
};
let start = end.saturating_sub(lookback);
(start..end)
.filter_map(|index| self.numeric_value_at(index, field))
.collect()
}
fn decision_price_on_or_before(&self, date: NaiveDate) -> Option<f64> {
let end = self.decision_end_index(date)?;
if end == 0 {
return None;
}
self.prev_closes.get(end - 1).copied()
}
fn decision_end_index(&self, date: NaiveDate) -> Option<usize> {
match self.dates.binary_search(&date) {
Ok(idx) => Some(idx + 1),
Err(0) => None,
Err(idx) => Some(idx),
}
}
fn previous_completed_end_index(&self, date: NaiveDate) -> Option<usize> {
match self.dates.binary_search(&date) {
Ok(idx) => Some(idx),
Err(0) => None,
Err(idx) => Some(idx),
}
}
fn decision_close_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = self.decision_end_index(date)?;
if end < lookback {
return None;
}
let start = end - lookback;
let sum = self.prev_close_prefix[end] - self.prev_close_prefix[start];
Some(sum / lookback as f64)
}
fn decision_volume_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
let end = self.previous_completed_end_index(date)?;
self.decision_volume_moving_average_at_end(end, lookback)
}
fn decision_volume_moving_average_at_end(
&self,
end: usize,
lookback: usize,
) -> Option<f64> {
self.valid_volume_window(end, lookback).map(|(start, end)| {
normalize_rolling_factor(
(self.valid_volume_sum_prefix[end] - self.valid_volume_sum_prefix[start])
/ lookback as f64,
12,
)
})
}
fn current_volume_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
let end = self.end_index(date)?;
self.valid_volume_window(end, lookback).map(|(start, end)| {
normalize_rolling_factor(
(self.valid_volume_sum_prefix[end] - self.valid_volume_sum_prefix[start])
/ lookback as f64,
12,
)
})
}
fn decision_volume_values(&self, date: NaiveDate, lookback: usize) -> Option<Vec<f64>> {
let end = self.previous_completed_end_index(date)?;
self.valid_volume_values(end, lookback)
}
fn current_volume_values(&self, date: NaiveDate, lookback: usize) -> Option<Vec<f64>> {
let end = self.end_index(date)?;
self.valid_volume_values(end, lookback)
}
fn valid_volume_window(&self, end: usize, lookback: usize) -> Option<(usize, usize)> {
if lookback == 0 || end > self.volumes.len() {
return None;
}
let valid_count = *self.valid_volume_count_prefix.get(end)?;
if valid_count < lookback {
return None;
}
let target_count = valid_count - lookback;
let start = *self.valid_volume_start_by_count.get(target_count)?;
debug_assert!(start <= end);
Some((start, end))
}
fn valid_volume_values(&self, end: usize, lookback: usize) -> Option<Vec<f64>> {
let (start, end) = self.valid_volume_window(end, lookback)?;
let values = self.volumes[start..end]
.iter()
.filter(|value| **value > 0)
.map(|value| *value as f64)
.collect::<Vec<_>>();
(values.len() == lookback).then_some(values)
}
fn end_index(&self, date: NaiveDate) -> Option<usize> {
match self.dates.binary_search(&date) {
Ok(idx) => Some(idx + 1),
Err(0) => None,
Err(idx) => Some(idx),
}
}
fn price_values_for(&self, field: PriceField) -> &[f64] {
match field {
PriceField::DayOpen => &self.day_opens,
PriceField::Open => &self.opens,
PriceField::Close => &self.closes,
PriceField::Last => &self.last_prices,
}
}
fn price_on_or_before(&self, date: NaiveDate, field: PriceField) -> Option<f64> {
let end = self.end_index(date)?;
if end == 0 {
return None;
}
self.price_values_for(field).get(end - 1).copied()
}
fn prefix_for(&self, field: PriceField) -> &[f64] {
match field {
PriceField::DayOpen => &self.open_prefix,
PriceField::Open => &self.open_prefix,
PriceField::Close => &self.close_prefix,
PriceField::Last => &self.last_prefix,
}
}
fn snapshot_at(&self, index: usize) -> DailyMarketSnapshot {
DailyMarketSnapshot {
date: self.dates[index],
symbol: self.symbol.clone(),
timestamp: self.timestamps[index].clone(),
day_open: self.day_opens[index],
open: self.opens[index],
high: self.highs[index],
low: self.lows[index],
close: self.closes[index],
last_price: self.last_prices[index],
bid1: self.bid1s[index],
ask1: self.ask1s[index],
prev_close: self.prev_closes[index],
volume: self.volumes[index],
minute_volume: self.minute_volumes[index],
bid1_volume: self.bid1_volumes[index],
ask1_volume: self.ask1_volumes[index],
trading_phase: self.trading_phases[index].clone(),
paused: self.paused[index],
upper_limit: self.upper_limits[index],
lower_limit: self.lower_limits[index],
price_tick: self.price_ticks[index],
}
}
fn numeric_value_at(&self, index: usize, field: &str) -> Option<f64> {
match normalized_field(field).as_ref() {
"day_open" | "dayopen" => Some(self.day_opens[index]),
"open" => Some(self.opens[index]),
"high" => Some(self.highs[index]),
"low" => Some(self.lows[index]),
"close" | "price" => Some(self.closes[index]),
"last" | "last_price" => Some(self.last_prices[index]),
"prev_close" | "pre_close" => Some(self.prev_closes[index]),
"volume" => Some(self.volumes[index] as f64),
"minute_volume" => Some(self.minute_volumes[index] as f64),
"bid1" => Some(self.bid1s[index]),
"ask1" => Some(self.ask1s[index]),
"bid1_volume" => Some(self.bid1_volumes[index] as f64),
"ask1_volume" => Some(self.ask1_volumes[index] as f64),
"upper_limit" => Some(self.upper_limits[index]),
"lower_limit" => Some(self.lower_limits[index]),
"price_tick" => Some(self.price_ticks[index]),
_ => None,
}
}
}
#[derive(Debug, Clone)]
struct BenchmarkPriceSeries {
dates: Vec<NaiveDate>,
opens: Vec<f64>,
closes: Vec<f64>,
prev_closes: Vec<f64>,
open_prefix: Vec<f64>,
close_prefix: Vec<f64>,
}
impl BenchmarkPriceSeries {
fn new(rows: &[BenchmarkSnapshot]) -> Self {
let mut sorted = rows.to_vec();
sorted.sort_by_key(|row| row.date);
let dates = sorted.iter().map(|row| row.date).collect::<Vec<_>>();
let opens = sorted.iter().map(|row| row.open).collect::<Vec<_>>();
let closes = sorted.iter().map(|row| row.close).collect::<Vec<_>>();
let prev_closes = sorted.iter().map(|row| row.prev_close).collect::<Vec<_>>();
let open_prefix = prefix_sums(&opens);
let close_prefix = prefix_sums(&closes);
Self {
dates,
opens,
closes,
prev_closes,
open_prefix,
close_prefix,
}
}
fn moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
self.moving_average_for(date, lookback, PriceField::Close)
}
fn decision_close(&self, date: NaiveDate) -> Option<f64> {
match self.dates.binary_search(&date) {
Ok(idx) => self
.prev_closes
.get(idx)
.copied()
.filter(|value| value.is_finite() && *value > 0.0)
.or_else(|| {
idx.checked_sub(1)
.and_then(|prev| self.closes.get(prev).copied())
}),
Err(0) => None,
Err(idx) => idx
.checked_sub(1)
.and_then(|prev| self.closes.get(prev).copied()),
}
}
fn decision_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = match self.dates.binary_search(&date) {
Ok(idx) => idx,
Err(0) => return None,
Err(idx) => idx,
};
if end < lookback {
return None;
}
let start = end - lookback;
let sum = self.close_prefix[end] - self.close_prefix[start];
Some(sum / lookback as f64)
}
fn decision_values_for(&self, date: NaiveDate, lookback: usize, field: PriceField) -> Vec<f64> {
if lookback == 0 {
return Vec::new();
}
let end = match self.dates.binary_search(&date) {
Ok(idx) => idx,
Err(0) => return Vec::new(),
Err(idx) => idx,
};
let start = end.saturating_sub(lookback);
match field {
PriceField::DayOpen | PriceField::Open => self.opens[start..end].to_vec(),
PriceField::Close | PriceField::Last => self.closes[start..end].to_vec(),
}
}
fn moving_average_for(
&self,
date: NaiveDate,
lookback: usize,
field: PriceField,
) -> Option<f64> {
if lookback == 0 {
return None;
}
let end = match self.dates.binary_search(&date) {
Ok(idx) => idx + 1,
Err(0) => return None,
Err(idx) => idx,
};
if end < lookback {
return None;
}
let start = end - lookback;
let prefix = match field {
PriceField::DayOpen | PriceField::Open => &self.open_prefix,
PriceField::Close | PriceField::Last => &self.close_prefix,
};
let sum = prefix[end] - prefix[start];
Some(sum / lookback as f64)
}
fn trailing_values(&self, date: NaiveDate, lookback: usize) -> Vec<f64> {
self.trailing_values_for(date, lookback, PriceField::Close)
}
fn trailing_values_for(&self, date: NaiveDate, lookback: usize, field: PriceField) -> Vec<f64> {
let end = match self.dates.binary_search(&date) {
Ok(idx) => idx + 1,
Err(0) => return Vec::new(),
Err(idx) => idx,
};
let start = end.saturating_sub(lookback);
match field {
PriceField::DayOpen | PriceField::Open => self.opens[start..end].to_vec(),
PriceField::Close | PriceField::Last => self.closes[start..end].to_vec(),
}
}
}
#[derive(Debug, Clone)]
pub struct DataSet {
instruments: Arc<HashMap<String, Instrument>>,
calendar: Arc<TradingCalendar>,
market_by_date: Arc<BTreeMap<NaiveDate, Vec<DailyMarketSnapshot>>>,
market_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
factor_by_date: Arc<BTreeMap<NaiveDate, Vec<DailyFactorSnapshot>>>,
factor_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
factor_text_by_date: Arc<BTreeMap<NaiveDate, Vec<FactorTextValue>>>,
factor_text_index: Arc<HashMap<(NaiveDate, String, String), FactorTextValue>>,
candidate_by_date: Arc<BTreeMap<NaiveDate, Vec<CandidateEligibility>>>,
candidate_symbol_ids_by_date: Arc<BTreeMap<NaiveDate, Vec<u32>>>,
corporate_actions_by_date: Arc<BTreeMap<NaiveDate, Vec<CorporateAction>>>,
execution_quotes_by_date: Arc<HashMap<NaiveDate, HashMap<String, Vec<IntradayExecutionQuote>>>>,
execution_quote_dates: Arc<Vec<NaiveDate>>,
order_book_depth_index: Arc<HashMap<(NaiveDate, String), Vec<IntradayOrderBookDepthLevel>>>,
benchmark_by_date: Arc<BTreeMap<NaiveDate, BenchmarkSnapshot>>,
market_series_by_symbol: Arc<AHashMap<String, Arc<SymbolPriceSeries>>>,
adjusted_close_series_by_symbol: Arc<AHashMap<String, Arc<AdjustedCloseSeries>>>,
market_series_by_symbol_id: Arc<Vec<Option<Arc<SymbolPriceSeries>>>>,
adjusted_close_series_by_symbol_id: Arc<Vec<Option<Arc<AdjustedCloseSeries>>>>,
benchmark_series_cache: Arc<BenchmarkPriceSeries>,
symbol_id_by_code: Arc<AHashMap<String, u32>>,
eligible_universe_by_date: Arc<OnceLock<BTreeMap<NaiveDate, Vec<EligibleUniverseSnapshot>>>>,
benchmark_code: String,
futures_params_by_symbol: Arc<HashMap<String, Vec<FuturesTradingParameter>>>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct SymbolSnapshotRefs<'a> {
pub market: Option<&'a DailyMarketSnapshot>,
pub factor: Option<&'a DailyFactorSnapshot>,
pub candidate: Option<&'a CandidateEligibility>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct DecisionRollingCursor<'a> {
adjusted_close: Option<(&'a AdjustedCloseSeries, usize)>,
volume: Option<(&'a SymbolPriceSeries, usize)>,
}
impl DecisionRollingCursor<'_> {
pub(crate) fn moving_average(&self, field: &str, lookback: usize) -> Option<f64> {
match field {
"close" => self
.adjusted_close
.and_then(|(series, end)| series.decision_moving_average_at_end(end, lookback)),
"volume" => self
.volume
.and_then(|(series, end)| {
series.decision_volume_moving_average_at_end(end, lookback)
}),
_ => None,
}
}
}
impl DataSet {
pub fn with_additional_trading_dates(
mut self,
dates: impl IntoIterator<Item = NaiveDate>,
) -> Self {
let mut calendar_dates = self.calendar.days().to_vec();
calendar_dates.extend(dates);
self.calendar = Arc::new(TradingCalendar::new(calendar_dates));
self
}
pub fn from_components(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_and_quotes(
instruments,
market,
factors,
candidates,
benchmarks,
Vec::new(),
Vec::new(),
)
}
pub fn from_components_with_actions(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_and_quotes(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
Vec::new(),
)
}
pub fn from_components_with_actions_and_quotes(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
execution_quotes: Vec<IntradayExecutionQuote>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_quotes_and_futures(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
Vec::new(),
)
}
pub fn from_components_with_actions_quotes_and_futures(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
execution_quotes: Vec<IntradayExecutionQuote>,
futures_params: Vec<FuturesTradingParameter>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_quotes_futures_and_depth(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
futures_params,
Vec::new(),
)
}
pub fn from_components_with_actions_quotes_futures_and_depth(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
execution_quotes: Vec<IntradayExecutionQuote>,
futures_params: Vec<FuturesTradingParameter>,
order_book_depth: Vec<IntradayOrderBookDepthLevel>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_quotes_futures_depth_and_factor_texts(
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
futures_params,
order_book_depth,
Vec::new(),
)
}
pub fn from_components_with_factor_texts(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
factor_texts: Vec<FactorTextValue>,
) -> Result<Self, DataSetError> {
Self::from_components_with_actions_quotes_futures_depth_and_factor_texts(
instruments,
market,
factors,
candidates,
benchmarks,
Vec::new(),
Vec::new(),
Vec::new(),
Vec::new(),
factor_texts,
)
}
pub fn from_components_with_actions_quotes_futures_depth_and_factor_texts(
instruments: Vec<Instrument>,
market: Vec<DailyMarketSnapshot>,
factors: Vec<DailyFactorSnapshot>,
candidates: Vec<CandidateEligibility>,
benchmarks: Vec<BenchmarkSnapshot>,
corporate_actions: Vec<CorporateAction>,
execution_quotes: Vec<IntradayExecutionQuote>,
futures_params: Vec<FuturesTradingParameter>,
order_book_depth: Vec<IntradayOrderBookDepthLevel>,
factor_texts: Vec<FactorTextValue>,
) -> Result<Self, DataSetError> {
let benchmark_code = collect_benchmark_code(&benchmarks)?;
let calendar = TradingCalendar::new(benchmarks.iter().map(|item| item.date).collect());
let factors = normalize_factor_snapshots(factors);
let instruments = instruments
.into_iter()
.map(|instrument| (instrument.symbol.clone(), instrument))
.collect::<HashMap<_, _>>();
let mut market_by_date = group_by_date(market, |item| item.date);
sort_groups_by_symbol(&mut market_by_date, |item| item.symbol.as_str());
let mut factor_by_date = group_by_date(factors, |item| item.date);
sort_groups_by_symbol(&mut factor_by_date, |item| item.symbol.as_str());
let mut market_rows_by_symbol = AHashMap::<String, Vec<&DailyMarketSnapshot>>::new();
for row in market_by_date.values().flatten() {
if let Some(rows) = market_rows_by_symbol.get_mut(row.symbol.as_str()) {
rows.push(row);
continue;
}
market_rows_by_symbol.insert(row.symbol.clone(), vec![row]);
}
let market_rows_by_symbol = market_rows_by_symbol.into_iter().collect::<Vec<_>>();
let market_series_by_symbol = market_rows_by_symbol
.into_par_iter()
.map(|(symbol, rows)| {
let series = Arc::new(SymbolPriceSeries::from_sorted_rows(symbol.clone(), rows));
(symbol, series)
})
.collect::<Vec<_>>()
.into_iter()
.collect::<AHashMap<_, _>>();
let mut factor_rows_by_symbol = AHashMap::<&str, Vec<&DailyFactorSnapshot>>::new();
for row in factor_by_date.values().flatten() {
factor_rows_by_symbol
.entry(row.symbol.as_str())
.or_default()
.push(row);
}
let adjusted_close_series_by_symbol = market_series_by_symbol
.par_iter()
.filter_map(|(symbol, market)| {
let factor_rows = factor_rows_by_symbol
.get(symbol.as_str())
.map(Vec::as_slice)
.unwrap_or_default();
AdjustedCloseSeries::new(market, factor_rows)
.map(|series| (symbol.clone(), Arc::new(series)))
})
.collect::<Vec<_>>()
.into_iter()
.collect::<AHashMap<_, _>>();
let factor_texts = factor_texts
.into_iter()
.filter_map(|mut item| {
item.field = normalize_field(&item.field);
if item.field.is_empty() {
None
} else {
Some(item)
}
})
.collect::<Vec<_>>();
let factor_text_by_date = group_by_date(factor_texts.clone(), |item| item.date);
let factor_text_index = factor_texts
.into_iter()
.map(|item| ((item.date, item.symbol.clone(), item.field.clone()), item))
.collect::<HashMap<_, _>>();
let mut candidate_by_date = group_by_date(candidates, |item| item.date);
sort_groups_by_symbol(&mut candidate_by_date, |item| item.symbol.as_str());
let symbol_id_by_code = build_symbol_id_index(
&instruments,
&market_by_date,
&factor_by_date,
&candidate_by_date,
);
let market_symbol_ids_by_date =
build_group_symbol_ids(&market_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str()
});
let factor_symbol_ids_by_date =
build_group_symbol_ids(&factor_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str()
});
let candidate_symbol_ids_by_date =
build_group_symbol_ids(&candidate_by_date, &symbol_id_by_code, |item| {
item.symbol.as_str()
});
let mut market_series_by_symbol_id = vec![None; symbol_id_by_code.len()];
for (symbol, series) in &market_series_by_symbol {
if let Some(symbol_id) = symbol_id_by_code.get(symbol).copied() {
market_series_by_symbol_id[symbol_id as usize] = Some(Arc::clone(series));
}
}
let mut adjusted_close_series_by_symbol_id = vec![None; symbol_id_by_code.len()];
for (symbol, series) in &adjusted_close_series_by_symbol {
if let Some(symbol_id) = symbol_id_by_code.get(symbol).copied() {
adjusted_close_series_by_symbol_id[symbol_id as usize] = Some(Arc::clone(series));
}
}
let corporate_actions_by_date = group_by_date(corporate_actions, |item| item.date);
let execution_quotes_by_date = build_execution_quote_index(execution_quotes);
let mut execution_quote_dates = execution_quotes_by_date.keys().copied().collect::<Vec<_>>();
execution_quote_dates.sort_unstable();
let order_book_depth_index = build_order_book_depth_index(order_book_depth);
let benchmark_by_date = benchmarks
.into_iter()
.map(|item| (item.date, item))
.collect::<BTreeMap<_, _>>();
let benchmark_series_cache =
BenchmarkPriceSeries::new(&benchmark_by_date.values().cloned().collect::<Vec<_>>());
let futures_params_by_symbol = build_futures_params_index(futures_params);
Ok(Self {
instruments: Arc::new(instruments),
calendar: Arc::new(calendar),
market_by_date: Arc::new(market_by_date),
market_symbol_ids_by_date: Arc::new(market_symbol_ids_by_date),
factor_by_date: Arc::new(factor_by_date),
factor_symbol_ids_by_date: Arc::new(factor_symbol_ids_by_date),
factor_text_by_date: Arc::new(factor_text_by_date),
factor_text_index: Arc::new(factor_text_index),
candidate_by_date: Arc::new(candidate_by_date),
candidate_symbol_ids_by_date: Arc::new(candidate_symbol_ids_by_date),
corporate_actions_by_date: Arc::new(corporate_actions_by_date),
execution_quotes_by_date: Arc::new(execution_quotes_by_date),
execution_quote_dates: Arc::new(execution_quote_dates),
order_book_depth_index: Arc::new(order_book_depth_index),
benchmark_by_date: Arc::new(benchmark_by_date),
market_series_by_symbol: Arc::new(market_series_by_symbol),
adjusted_close_series_by_symbol: Arc::new(adjusted_close_series_by_symbol),
market_series_by_symbol_id: Arc::new(market_series_by_symbol_id),
adjusted_close_series_by_symbol_id: Arc::new(adjusted_close_series_by_symbol_id),
benchmark_series_cache: Arc::new(benchmark_series_cache),
symbol_id_by_code: Arc::new(symbol_id_by_code),
eligible_universe_by_date: Arc::new(OnceLock::new()),
benchmark_code,
futures_params_by_symbol: Arc::new(futures_params_by_symbol),
})
}
pub fn calendar(&self) -> &TradingCalendar {
&self.calendar
}
pub fn benchmark_code(&self) -> &str {
&self.benchmark_code
}
pub fn instruments(&self) -> &HashMap<String, Instrument> {
&self.instruments
}
pub fn all_instruments(&self) -> Vec<&Instrument> {
let mut instruments = self.instruments.values().collect::<Vec<_>>();
instruments.sort_by(|left, right| left.symbol.cmp(&right.symbol));
instruments
}
pub fn instruments_history(&self, symbols: &[&str]) -> Vec<&Instrument> {
symbols
.iter()
.filter_map(|symbol| self.instruments.get(*symbol))
.collect()
}
pub fn active_instruments(&self, date: NaiveDate, symbols: &[&str]) -> Vec<&Instrument> {
symbols
.iter()
.filter_map(|symbol| self.instruments.get(*symbol))
.filter(|instrument| instrument.is_active_on(date))
.collect()
}
pub fn instrument(&self, symbol: &str) -> Option<&Instrument> {
self.instruments.get(symbol)
}
pub fn symbol_id(&self, symbol: &str) -> Option<u32> {
self.symbol_id_by_code.get(symbol).copied()
}
pub fn market(&self, date: NaiveDate, symbol: &str) -> Option<&DailyMarketSnapshot> {
let symbol_id = self.symbol_id(symbol)?;
self.market_by_symbol_id(date, symbol_id)
}
pub fn market_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
) -> Option<&DailyMarketSnapshot> {
find_by_symbol_id(
self.market_by_date.get(&date)?,
self.market_symbol_ids_by_date.get(&date)?,
symbol_id,
)
}
fn market_series(&self, symbol: &str) -> Option<&SymbolPriceSeries> {
self.market_series_by_symbol.get(symbol).map(Arc::as_ref)
}
fn market_series_by_symbol_id(&self, symbol_id: u32) -> Option<&SymbolPriceSeries> {
self.market_series_by_symbol_id
.get(symbol_id as usize)?
.as_deref()
}
fn adjusted_close_series(&self, symbol: &str) -> Option<&AdjustedCloseSeries> {
self.adjusted_close_series_by_symbol
.get(symbol)
.map(Arc::as_ref)
}
fn adjusted_close_series_by_symbol_id(&self, symbol_id: u32) -> Option<&AdjustedCloseSeries> {
self.adjusted_close_series_by_symbol_id
.get(symbol_id as usize)?
.as_deref()
}
pub fn factor(&self, date: NaiveDate, symbol: &str) -> Option<&DailyFactorSnapshot> {
let symbol_id = self.symbol_id(symbol)?;
self.factor_by_symbol_id(date, symbol_id)
}
pub fn factor_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
) -> Option<&DailyFactorSnapshot> {
find_by_symbol_id(
self.factor_by_date.get(&date)?,
self.factor_symbol_ids_by_date.get(&date)?,
symbol_id,
)
}
pub fn candidate(&self, date: NaiveDate, symbol: &str) -> Option<&CandidateEligibility> {
let symbol_id = self.symbol_id(symbol)?;
self.candidate_by_symbol_id(date, symbol_id)
}
pub fn candidate_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
) -> Option<&CandidateEligibility> {
find_by_symbol_id(
self.candidate_by_date.get(&date)?,
self.candidate_symbol_ids_by_date.get(&date)?,
symbol_id,
)
}
pub(crate) fn symbol_snapshots_by_id(
&self,
date: NaiveDate,
symbol_id: u32,
) -> SymbolSnapshotRefs<'_> {
let market_rows = self.market_by_date.get(&date).map(Vec::as_slice);
let market_symbol_ids = self
.market_symbol_ids_by_date
.get(&date)
.map(Vec::as_slice);
let market_index = market_rows
.zip(market_symbol_ids)
.and_then(|(rows, symbol_ids)| symbol_id_index(rows.len(), symbol_ids, symbol_id));
let market = market_index.and_then(|index| market_rows?.get(index));
let factor = self
.factor_by_date
.get(&date)
.map(Vec::as_slice)
.zip(
self.factor_symbol_ids_by_date
.get(&date)
.map(Vec::as_slice),
)
.and_then(|(rows, symbol_ids)| {
find_by_symbol_id_with_preferred_index(
rows,
symbol_ids,
symbol_id,
market_index,
)
});
let candidate = self
.candidate_by_date
.get(&date)
.map(Vec::as_slice)
.zip(
self.candidate_symbol_ids_by_date
.get(&date)
.map(Vec::as_slice),
)
.and_then(|(rows, symbol_ids)| {
find_by_symbol_id_with_preferred_index(
rows,
symbol_ids,
symbol_id,
market_index,
)
});
SymbolSnapshotRefs {
market,
factor,
candidate,
}
}
pub fn benchmark(&self, date: NaiveDate) -> Option<&BenchmarkSnapshot> {
self.benchmark_by_date.get(&date)
}
pub fn corporate_actions_on(&self, date: NaiveDate) -> &[CorporateAction] {
self.corporate_actions_by_date
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn execution_quotes_on(&self, date: NaiveDate, symbol: &str) -> &[IntradayExecutionQuote] {
self.execution_quotes_by_date
.get(&date)
.and_then(|rows_by_symbol| rows_by_symbol.get(symbol))
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn has_execution_quotes_on_date(&self, date: NaiveDate) -> bool {
self.execution_quotes_by_date
.get(&date)
.map(|rows_by_symbol| !rows_by_symbol.is_empty())
.unwrap_or(false)
}
pub fn execution_quote_key_set(&self) -> HashSet<(NaiveDate, String)> {
self.execution_quotes_by_date
.iter()
.flat_map(|(date, rows_by_symbol)| {
rows_by_symbol
.keys()
.map(move |symbol| (*date, symbol.clone()))
})
.collect()
}
pub fn execution_quote_count(&self) -> usize {
self.execution_quotes_by_date
.values()
.flat_map(|rows_by_symbol| rows_by_symbol.values())
.map(Vec::len)
.sum()
}
pub fn add_execution_quotes(&mut self, quotes: Vec<IntradayExecutionQuote>) -> usize {
let mut grouped = HashMap::<NaiveDate, HashMap<String, Vec<IntradayExecutionQuote>>>::new();
for quote in quotes {
grouped
.entry(quote.date)
.or_default()
.entry(quote.symbol.clone())
.or_default()
.push(quote);
}
let mut added = 0usize;
let mut new_dates = Vec::new();
let execution_quotes_by_date = Arc::make_mut(&mut self.execution_quotes_by_date);
for (date, rows_by_symbol) in grouped {
let date_is_new = !execution_quotes_by_date.contains_key(&date);
let target_by_symbol = execution_quotes_by_date.entry(date).or_default();
if date_is_new {
new_dates.push(date);
}
for (symbol, mut incoming) in rows_by_symbol {
incoming.sort_by_key(|quote| quote.timestamp);
incoming.dedup_by(|left, right| left.timestamp == right.timestamp);
let target = target_by_symbol.entry(symbol).or_default();
if target.is_empty() {
added = added.saturating_add(incoming.len());
*target = incoming;
continue;
}
let mut existing = std::mem::take(target).into_iter().peekable();
let mut incoming = incoming.into_iter().peekable();
let mut merged = Vec::with_capacity(existing.len() + incoming.len());
while let (Some(existing_quote), Some(incoming_quote)) =
(existing.peek(), incoming.peek())
{
match existing_quote.timestamp.cmp(&incoming_quote.timestamp) {
std::cmp::Ordering::Less => {
merged.push(existing.next().expect("peeked existing quote"));
}
std::cmp::Ordering::Greater => {
merged.push(incoming.next().expect("peeked incoming quote"));
added = added.saturating_add(1);
}
std::cmp::Ordering::Equal => {
merged.push(existing.next().expect("peeked existing quote"));
incoming.next();
}
}
}
merged.extend(existing);
for quote in incoming {
merged.push(quote);
added = added.saturating_add(1);
}
*target = merged;
}
}
if !new_dates.is_empty() {
let dates = Arc::make_mut(&mut self.execution_quote_dates);
for date in new_dates {
if let Err(index) = dates.binary_search(&date) {
dates.insert(index, date);
}
}
}
added
}
pub fn order_book_depth_on(
&self,
date: NaiveDate,
symbol: &str,
) -> &[IntradayOrderBookDepthLevel] {
self.order_book_depth_index
.get(&(date, symbol.to_string()))
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn execution_quotes_on_date(&self, date: NaiveDate) -> Vec<IntradayExecutionQuote> {
self.execution_quotes_on_date_for_symbols(date, None)
}
pub fn execution_quotes_iter_on_date_for_symbols(
&self,
date: NaiveDate,
symbols: Option<&BTreeSet<String>>,
) -> ExecutionQuoteIterator<'_> {
ExecutionQuoteIterator::new(self.execution_quotes_by_date.get(&date), symbols)
}
pub fn execution_quotes_on_date_for_symbols(
&self,
date: NaiveDate,
symbols: Option<&BTreeSet<String>>,
) -> Vec<IntradayExecutionQuote> {
self.execution_quotes_iter_on_date_for_symbols(date, symbols)
.cloned()
.collect()
}
pub fn remove_execution_quotes_on_date(&mut self, date: NaiveDate) -> usize {
let removed = Arc::make_mut(&mut self.execution_quotes_by_date).remove(&date);
let Some(rows_by_symbol) = removed else {
return 0;
};
let dates = Arc::make_mut(&mut self.execution_quote_dates);
if let Ok(index) = dates.binary_search(&date) {
dates.remove(index);
}
rows_by_symbol
.into_values()
.map(|rows| rows.len())
.sum()
}
pub fn snapshot_components(&self) -> DataSetSnapshotComponents {
let mut instruments = self.instruments.values().cloned().collect::<Vec<_>>();
instruments.sort_by(|left, right| left.symbol.cmp(&right.symbol));
let market = self
.market_by_date
.values()
.flat_map(|rows| rows.iter().cloned())
.collect::<Vec<_>>();
let factors = self
.factor_by_date
.values()
.flat_map(|rows| rows.iter().cloned())
.collect::<Vec<_>>();
let candidates = self
.candidate_by_date
.values()
.flat_map(|rows| rows.iter().cloned())
.collect::<Vec<_>>();
let benchmarks = self.benchmark_by_date.values().cloned().collect::<Vec<_>>();
let corporate_actions = self
.corporate_actions_by_date
.values()
.flat_map(|rows| rows.iter().cloned())
.collect::<Vec<_>>();
let execution_quotes = self
.execution_quotes_by_date
.values()
.flat_map(|rows_by_symbol| rows_by_symbol.values())
.flat_map(|rows| rows.iter().cloned())
.collect::<Vec<_>>();
DataSetSnapshotComponents {
instruments,
market,
factors,
candidates,
benchmarks,
corporate_actions,
execution_quotes,
}
}
pub fn benchmark_series(&self) -> Vec<BenchmarkSnapshot> {
self.benchmark_by_date.values().cloned().collect()
}
pub fn futures_trading_parameter(
&self,
date: NaiveDate,
symbol: &str,
) -> Option<&FuturesTradingParameter> {
self.futures_params_by_symbol.get(symbol).and_then(|rows| {
rows.iter()
.rev()
.find(|row| row.effective_date.is_none_or(|effective| effective <= date))
})
}
pub fn futures_settlement_price(
&self,
date: NaiveDate,
symbol: &str,
mode: &str,
) -> Option<f64> {
let snapshot = self.market(date, symbol)?;
match normalize_field(mode).as_str() {
"settlement" | "settle" => self
.factor_numeric_value(date, symbol, "settlement")
.or_else(|| self.factor_numeric_value(date, symbol, "settle"))
.or(Some(snapshot.close)),
"prev_settlement" | "pre_settlement" => self
.factor_numeric_value(date, symbol, "prev_settlement")
.or_else(|| self.factor_numeric_value(date, symbol, "pre_settlement"))
.or(Some(snapshot.prev_close)),
_ => Some(snapshot.close),
}
}
pub fn history_bars(
&self,
date: NaiveDate,
symbol: &str,
bar_count: usize,
frequency: &str,
field: &str,
include_now: bool,
) -> Vec<f64> {
self.history_bars_at(date, None, symbol, bar_count, frequency, field, include_now)
}
pub fn history_bars_at(
&self,
date: NaiveDate,
active_datetime: Option<NaiveDateTime>,
symbol: &str,
bar_count: usize,
frequency: &str,
field: &str,
include_now: bool,
) -> Vec<f64> {
if bar_count == 0 {
return Vec::new();
}
match normalize_history_frequency(frequency).as_deref() {
Some("1d") => self.history_daily_values(date, symbol, bar_count, field, include_now),
Some("1m") => self.history_intraday_values(
date,
active_datetime,
symbol,
bar_count,
field,
include_now,
),
_ => Vec::new(),
}
}
pub fn history_daily_snapshots(
&self,
date: NaiveDate,
symbol: &str,
bar_count: usize,
include_now: bool,
) -> Vec<DailyMarketSnapshot> {
self.market_series(symbol)
.map(|series| series.trailing_snapshots(date, bar_count, include_now))
.unwrap_or_default()
}
pub fn history_intraday_quotes(
&self,
date: NaiveDate,
symbol: &str,
bar_count: usize,
include_now: bool,
) -> Vec<IntradayExecutionQuote> {
self.history_intraday_quotes_at(date, None, symbol, bar_count, include_now)
}
pub fn history_intraday_quotes_at(
&self,
date: NaiveDate,
active_datetime: Option<NaiveDateTime>,
symbol: &str,
bar_count: usize,
include_now: bool,
) -> Vec<IntradayExecutionQuote> {
if bar_count == 0 {
return Vec::new();
}
let end = self
.execution_quote_dates
.partition_point(|quote_date| *quote_date <= date);
let mut quotes = Vec::with_capacity(bar_count);
'dates: for quote_date in self.execution_quote_dates[..end].iter().rev() {
let Some(rows) = self
.execution_quotes_by_date
.get(quote_date)
.and_then(|rows_by_symbol| rows_by_symbol.get(symbol))
else {
continue;
};
for quote in rows.iter().rev() {
if intraday_quote_visible(quote, date, active_datetime, include_now) {
quotes.push(quote.clone());
if quotes.len() == bar_count {
break 'dates;
}
}
}
}
quotes.reverse();
quotes
}
pub fn trading_dates(&self, start: NaiveDate, end: NaiveDate) -> Vec<NaiveDate> {
self.calendar.trading_dates(start, end)
}
pub fn previous_trading_date(&self, date: NaiveDate, n: usize) -> Option<NaiveDate> {
self.calendar.previous_trading_date(date, n)
}
pub fn next_trading_date(&self, date: NaiveDate, n: usize) -> Option<NaiveDate> {
self.calendar.next_trading_date(date, n)
}
pub fn is_suspended_flags(&self, date: NaiveDate, symbol: &str, count: usize) -> Vec<bool> {
self.historical_daily_flags(date, symbol, count, |candidate, market| {
candidate.is_some_and(|row| row.is_paused) || market.is_some_and(|row| row.paused)
})
}
pub fn is_st_stock_flags(&self, date: NaiveDate, symbol: &str, count: usize) -> Vec<bool> {
self.historical_daily_flags(date, symbol, count, |candidate, _| {
candidate.is_some_and(|row| row.is_st)
})
}
pub fn get_dividend(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
) -> Vec<DividendRecord> {
let mut rows = self
.corporate_actions_by_date
.range(start..=end)
.flat_map(|(_, actions)| actions.iter())
.filter(|action| action.symbol == symbol && action.share_cash.abs() > f64::EPSILON)
.map(|action| DividendRecord {
ex_dividend_date: action.date,
payable_date: action.payable_date.unwrap_or(action.date),
symbol: action.symbol.clone(),
dividend_cash_before_tax: action.share_cash,
round_lot: self
.instrument(symbol)
.map(Instrument::effective_round_lot)
.unwrap_or(100),
})
.collect::<Vec<_>>();
rows.sort_by_key(|row| row.ex_dividend_date);
rows
}
pub fn get_split(&self, symbol: &str, start: NaiveDate, end: NaiveDate) -> Vec<SplitRecord> {
let mut rows = self
.corporate_actions_by_date
.range(start..=end)
.flat_map(|(_, actions)| actions.iter())
.filter(|action| action.symbol == symbol && (action.split_ratio() - 1.0).abs() > 1e-12)
.map(|action| SplitRecord {
ex_dividend_date: action.date,
symbol: action.symbol.clone(),
split_ratio: action.split_ratio(),
})
.collect::<Vec<_>>();
rows.sort_by_key(|row| row.ex_dividend_date);
rows
}
pub fn get_factor(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
if start > end {
return Vec::new();
}
let field = normalize_field(field);
let mut rows = self
.factor_by_date
.range(start..=end)
.flat_map(|(_, snapshots)| snapshots.iter())
.filter(|snapshot| snapshot.symbol == symbol)
.filter_map(|snapshot| {
factor_numeric_value(snapshot, &field).map(|value| FactorValue {
date: snapshot.date,
symbol: snapshot.symbol.clone(),
field: field.clone(),
value,
})
})
.collect::<Vec<_>>();
rows.sort_by_key(|row| row.date);
rows
}
pub fn get_factor_text(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorTextValue> {
if start > end {
return Vec::new();
}
let field = normalize_field(field);
let mut rows = self
.factor_text_by_date
.range(start..=end)
.flat_map(|(_, snapshots)| snapshots.iter())
.filter(|snapshot| {
snapshot.symbol == symbol && normalize_field(&snapshot.field) == field
})
.cloned()
.collect::<Vec<_>>();
rows.sort_by_key(|row| row.date);
rows
}
pub fn get_yield_curve(
&self,
start: NaiveDate,
end: NaiveDate,
tenor: Option<&str>,
) -> Vec<YieldCurvePoint> {
if start > end {
return Vec::new();
}
let tenor_filter = tenor.map(normalize_field);
let mut rows = Vec::new();
for (date, snapshots) in self.factor_by_date.range(start..=end) {
for snapshot in snapshots {
for (field, value) in &snapshot.extra_factors {
let normalized = normalize_field(field);
let Some(raw_tenor) = normalized
.strip_prefix("yield_curve_")
.or_else(|| normalized.strip_prefix("yc_"))
else {
continue;
};
if tenor_filter
.as_ref()
.is_some_and(|expected| expected != raw_tenor)
{
continue;
}
rows.push(YieldCurvePoint {
date: *date,
tenor: raw_tenor.to_string(),
value: *value,
});
}
}
}
rows.sort_by(|left, right| {
left.date
.cmp(&right.date)
.then(left.tenor.cmp(&right.tenor))
});
rows
}
pub fn get_margin_stocks(&self, date: NaiveDate, margin_type: &str) -> Vec<String> {
let field = match normalize_field(margin_type).as_str() {
"stock" => "margin_stock",
"cash" => "margin_cash",
_ => "margin_all",
};
let mut symbols = self
.factor_by_date
.get(&date)
.map(|rows| {
rows.iter()
.filter(|row| {
row.extra_factors
.get(field)
.or_else(|| row.extra_factors.get("margin_all"))
.is_some_and(|value| *value > 0.0)
})
.map(|row| row.symbol.clone())
.collect::<Vec<_>>()
})
.unwrap_or_default();
if symbols.is_empty() {
symbols = self
.active_instruments(
date,
&self
.instruments
.keys()
.map(String::as_str)
.collect::<Vec<_>>(),
)
.into_iter()
.filter(|instrument| !instrument.board.eq_ignore_ascii_case("FUTURE"))
.map(|instrument| instrument.symbol.clone())
.collect();
}
symbols.sort();
symbols.dedup();
symbols
}
pub fn get_securities_margin(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<SecuritiesMarginRecord> {
self.get_factor(symbol, start, end, field)
.into_iter()
.map(|row| SecuritiesMarginRecord {
date: row.date,
symbol: row.symbol,
field: row.field,
value: row.value,
})
.collect()
}
pub fn get_shares(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
share_type: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&shares_factor_aliases(share_type),
&format!("shares_{}", normalize_field(share_type)),
)
}
pub fn get_turnover_rate(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&turnover_rate_factor_aliases(field),
&format!("turnover_rate_{}", normalize_field(field)),
)
}
pub fn get_price_change_rate(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
) -> Vec<FactorValue> {
if start > end {
return Vec::new();
}
let mut rows = self
.market_by_date
.range(start..=end)
.flat_map(|(_, snapshots)| snapshots.iter())
.filter(|snapshot| snapshot.symbol == symbol)
.filter_map(|snapshot| {
if snapshot.prev_close.is_finite() && snapshot.prev_close > 0.0 {
Some(FactorValue {
date: snapshot.date,
symbol: snapshot.symbol.clone(),
field: "price_change_rate".to_string(),
value: snapshot.close / snapshot.prev_close - 1.0,
})
} else {
None
}
})
.collect::<Vec<_>>();
if rows.is_empty() {
rows = self.get_first_available_factor_series(
symbol,
start,
end,
&[
"price_change_rate".to_string(),
"change_rate".to_string(),
"pct_change".to_string(),
],
"price_change_rate",
);
}
rows.sort_by_key(|row| row.date);
rows
}
pub fn get_stock_connect(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&stock_connect_factor_aliases(field),
&format!("stock_connect_{}", normalize_field(field)),
)
}
pub fn current_performance(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&prefixed_factor_aliases("current_performance", field),
field,
)
}
pub fn get_fundamentals(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&prefixed_factor_aliases("fundamental", field),
field,
)
}
pub fn get_financials(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&prefixed_factor_aliases("financial", field),
field,
)
}
pub fn get_pit_financials(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
field: &str,
) -> Vec<FactorValue> {
self.get_first_available_factor_series(
symbol,
start,
end,
&prefixed_factor_aliases("pit_financial", field),
field,
)
}
pub fn get_industry(
&self,
symbol: &str,
date: NaiveDate,
source: &str,
level: usize,
) -> Option<FactorValue> {
let fields = industry_factor_aliases(source, level);
for (factor_date, snapshots) in self.factor_by_date.range(..=date).rev() {
let Some(snapshot) = snapshots.iter().find(|row| row.symbol == symbol) else {
continue;
};
for field in &fields {
if let Some(value) = factor_numeric_value(snapshot, field) {
return Some(FactorValue {
date: *factor_date,
symbol: snapshot.symbol.clone(),
field: field.clone(),
value,
});
}
}
}
None
}
pub fn get_industry_name(
&self,
symbol: &str,
date: NaiveDate,
source: &str,
level: usize,
) -> Option<FactorTextValue> {
let fields = industry_name_factor_aliases(source, level);
for (factor_date, snapshots) in self.factor_text_by_date.range(..=date).rev() {
for snapshot in snapshots {
if snapshot.symbol != symbol {
continue;
}
let normalized = normalize_field(&snapshot.field);
if fields.iter().any(|field| field == &normalized) {
return Some(FactorTextValue {
date: *factor_date,
symbol: snapshot.symbol.clone(),
field: snapshot.field.clone(),
value: snapshot.value.clone(),
});
}
}
}
None
}
pub fn get_dominant_future(&self, underlying_symbol: &str, date: NaiveDate) -> Option<String> {
let underlying = normalize_field(underlying_symbol);
let mut candidates = self
.futures_params_by_symbol
.keys()
.filter(|symbol| normalize_field(symbol).starts_with(&underlying))
.filter(|symbol| {
self.futures_trading_parameter(date, symbol.as_str())
.is_some()
})
.cloned()
.collect::<Vec<_>>();
if candidates.is_empty() {
candidates = self
.instruments
.values()
.filter(|instrument| instrument.board.eq_ignore_ascii_case("FUTURE"))
.filter(|instrument| normalize_field(&instrument.symbol).starts_with(&underlying))
.filter(|instrument| instrument.is_active_on(date))
.map(|instrument| instrument.symbol.clone())
.collect();
}
candidates.sort();
candidates.into_iter().next()
}
pub fn get_dominant_future_price(
&self,
underlying_symbol: &str,
start: NaiveDate,
end: NaiveDate,
frequency: &str,
) -> Vec<PriceBar> {
let Some(symbol) = self.get_dominant_future(underlying_symbol, end) else {
return Vec::new();
};
self.get_price(&symbol, start, end, frequency)
}
pub fn get_price(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
frequency: &str,
) -> Vec<PriceBar> {
if start > end {
return Vec::new();
}
match normalize_history_frequency(frequency).as_deref() {
Some("1d") => self
.market_by_date
.range(start..=end)
.flat_map(|(_, rows)| rows.iter())
.filter(|row| row.symbol == symbol)
.map(daily_market_price_bar)
.collect(),
Some("1m") => {
let mut bars = self
.execution_quotes_by_date
.iter()
.filter(|(date, _)| **date >= start && **date <= end)
.filter_map(|(_, rows_by_symbol)| rows_by_symbol.get(symbol))
.flat_map(|rows| rows.iter())
.map(intraday_quote_price_bar)
.collect::<Vec<_>>();
bars.sort_by(|left, right| {
left.date
.cmp(&right.date)
.then_with(|| left.timestamp.cmp(&right.timestamp))
});
bars
}
_ => Vec::new(),
}
}
pub fn price(&self, date: NaiveDate, symbol: &str, field: PriceField) -> Option<f64> {
let snapshot = self.market(date, symbol)?;
Some(snapshot.price(field))
}
pub fn price_on_or_before(
&self,
date: NaiveDate,
symbol: &str,
field: PriceField,
) -> Option<f64> {
self.market_series(symbol)
.and_then(|series| series.price_on_or_before(date, field))
}
pub fn market_before(&self, date: NaiveDate, symbol: &str) -> Option<&DailyMarketSnapshot> {
let series = self.market_series(symbol)?;
let end = series.previous_completed_end_index(date)?;
if end == 0 {
return None;
}
let previous_date = *series.dates.get(end - 1)?;
self.market(previous_date, symbol)
}
pub fn factor_snapshots_on(&self, date: NaiveDate) -> Vec<&DailyFactorSnapshot> {
self.factor_by_date
.get(&date)
.map(|rows| rows.iter().collect())
.unwrap_or_default()
}
pub fn factor_snapshot_rows_on(&self, date: NaiveDate) -> &[DailyFactorSnapshot] {
self.factor_by_date
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn factor_symbol_ids_on(&self, date: NaiveDate) -> &[u32] {
self.factor_symbol_ids_by_date
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn factor_text_snapshots_on(&self, date: NaiveDate) -> Vec<&FactorTextValue> {
self.factor_text_by_date
.get(&date)
.map(|rows| rows.iter().collect())
.unwrap_or_default()
}
pub fn factor_text_rows_on(&self, date: NaiveDate) -> &[FactorTextValue] {
self.factor_text_by_date
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn market_snapshots_on(&self, date: NaiveDate) -> Vec<&DailyMarketSnapshot> {
self.market_by_date
.get(&date)
.map(|rows| rows.iter().collect())
.unwrap_or_default()
}
pub fn market_snapshot_rows_on(&self, date: NaiveDate) -> &[DailyMarketSnapshot] {
self.market_by_date
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn candidate_snapshots_on(&self, date: NaiveDate) -> Vec<&CandidateEligibility> {
self.candidate_by_date
.get(&date)
.map(|rows| rows.iter().collect())
.unwrap_or_default()
}
pub fn candidate_snapshot_rows_on(&self, date: NaiveDate) -> &[CandidateEligibility] {
self.candidate_by_date
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn bundle_on(&self, date: NaiveDate) -> Result<DailySnapshotBundle, DataSetError> {
let benchmark = self
.benchmark(date)
.cloned()
.ok_or(DataSetError::MissingBenchmark { date })?;
Ok(DailySnapshotBundle {
date,
benchmark,
market: self.market_by_date.get(&date).cloned().unwrap_or_default(),
factors: self.factor_by_date.get(&date).cloned().unwrap_or_default(),
candidates: self
.candidate_by_date
.get(&date)
.cloned()
.unwrap_or_default(),
corporate_actions: self
.corporate_actions_by_date
.get(&date)
.cloned()
.unwrap_or_default(),
})
}
pub fn benchmark_closes_up_to(&self, date: NaiveDate, lookback: usize) -> Vec<f64> {
self.benchmark_series_cache.trailing_values(date, lookback)
}
pub fn market_closes_up_to(&self, date: NaiveDate, symbol: &str, lookback: usize) -> Vec<f64> {
self.market_series(symbol)
.map(|series| series.trailing_values(date, lookback, PriceField::Close))
.unwrap_or_default()
}
fn history_daily_values(
&self,
date: NaiveDate,
symbol: &str,
bar_count: usize,
field: &str,
include_now: bool,
) -> Vec<f64> {
self.market_series(symbol)
.map(|series| series.trailing_numeric_values(date, bar_count, field, include_now))
.unwrap_or_default()
}
fn history_intraday_values(
&self,
date: NaiveDate,
active_datetime: Option<NaiveDateTime>,
symbol: &str,
bar_count: usize,
field: &str,
include_now: bool,
) -> Vec<f64> {
self.history_intraday_quotes_at(date, active_datetime, symbol, bar_count, include_now)
.into_iter()
.filter_map(|row| intraday_quote_numeric_value(&row, field))
.collect()
}
fn historical_daily_flags<F>(
&self,
date: NaiveDate,
symbol: &str,
count: usize,
evaluator: F,
) -> Vec<bool>
where
F: Fn(Option<&CandidateEligibility>, Option<&DailyMarketSnapshot>) -> bool,
{
if count == 0 {
return Vec::new();
}
let days = self
.calendar
.iter()
.filter(|day| *day <= date)
.collect::<Vec<_>>();
let start = days.len().saturating_sub(count);
days[start..]
.iter()
.map(|day| evaluator(self.candidate(*day, symbol), self.market(*day, symbol)))
.collect()
}
pub fn market_decision_close(&self, date: NaiveDate, symbol: &str) -> Option<f64> {
self.market_series(symbol)
.and_then(|series| series.decision_price_on_or_before(date))
}
pub fn market_decision_close_moving_average(
&self,
date: NaiveDate,
symbol: &str,
lookback: usize,
) -> Option<f64> {
self.market_series(symbol)
.and_then(|series| series.decision_close_moving_average(date, lookback))
}
pub fn market_decision_volume_moving_average(
&self,
date: NaiveDate,
symbol: &str,
lookback: usize,
) -> Option<f64> {
self.market_series(symbol)
.and_then(|series| series.decision_volume_moving_average(date, lookback))
}
pub fn factor_numeric_value(&self, date: NaiveDate, symbol: &str, field: &str) -> Option<f64> {
self.factor(date, symbol)
.and_then(|snapshot| factor_numeric_value(snapshot, field))
}
pub fn factor_text_value(&self, date: NaiveDate, symbol: &str, field: &str) -> Option<String> {
self.factor_text_index
.get(&(date, symbol.to_string(), normalize_field(field)))
.map(|row| row.value.clone())
}
fn get_first_available_factor_series(
&self,
symbol: &str,
start: NaiveDate,
end: NaiveDate,
fields: &[String],
output_field: &str,
) -> Vec<FactorValue> {
if start > end {
return Vec::new();
}
let output_field = normalize_field(output_field);
let mut rows = Vec::new();
for (_, snapshots) in self.factor_by_date.range(start..=end) {
let Some(snapshot) = snapshots.iter().find(|row| row.symbol == symbol) else {
continue;
};
for field in fields {
if let Some(value) = factor_numeric_value(snapshot, field) {
rows.push(FactorValue {
date: snapshot.date,
symbol: snapshot.symbol.clone(),
field: output_field.clone(),
value,
});
break;
}
}
}
rows.sort_by_key(|row| row.date);
rows
}
pub fn factor_moving_average(
&self,
date: NaiveDate,
symbol: &str,
field: &str,
lookback: usize,
) -> Option<f64> {
if lookback == 0 {
return None;
}
let dates = self.calendar.trailing_days(date, lookback);
if dates.is_empty() {
return None;
}
let mut sum = 0.0_f64;
let mut count = 0usize;
for trading_day in dates {
let snapshot = self.factor(trading_day, symbol)?;
let value = factor_numeric_value(snapshot, field)?;
sum += value;
count += 1;
}
if count == 0 {
None
} else {
Some(sum / count as f64)
}
}
pub fn market_decision_numeric_moving_average(
&self,
date: NaiveDate,
symbol: &str,
field: &str,
lookback: usize,
) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series(symbol)
.and_then(|series| series.decision_moving_average(date, lookback)),
"volume" | "stock_volume" => self
.market_series(symbol)
.and_then(|series| series.decision_volume_moving_average(date, lookback)),
"day_open" | "dayopen" => {
self.market_moving_average(date, symbol, lookback, PriceField::DayOpen)
}
"open" => self.market_moving_average(date, symbol, lookback, PriceField::Open),
"last" | "last_price" => {
self.market_moving_average(date, symbol, lookback, PriceField::Last)
}
other => self.factor_moving_average(date, symbol, other, lookback),
}
}
pub fn market_decision_numeric_moving_average_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
field: &str,
lookback: usize,
) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series_by_symbol_id(symbol_id)
.and_then(|series| series.decision_moving_average(date, lookback)),
"volume" | "stock_volume" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.decision_volume_moving_average(date, lookback)),
"day_open" | "dayopen" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.moving_average(date, lookback, PriceField::DayOpen)),
"open" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.moving_average(date, lookback, PriceField::Open)),
"last" | "last_price" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.moving_average(date, lookback, PriceField::Last)),
other => self.factor_moving_average(date, symbol, other, lookback),
}
}
pub(crate) fn market_decision_rolling_cursor_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
) -> DecisionRollingCursor<'_> {
let adjusted_close = self
.adjusted_close_series_by_symbol_id(symbol_id)
.and_then(|series| series.decision_end_index(date).map(|end| (series, end)));
let volume = self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| {
series
.previous_completed_end_index(date)
.map(|end| (series, end))
});
DecisionRollingCursor {
adjusted_close,
volume,
}
}
pub fn market_current_numeric_moving_average(
&self,
date: NaiveDate,
symbol: &str,
field: &str,
lookback: usize,
) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series(symbol)
.and_then(|series| series.current_moving_average(date, lookback)),
"volume" | "stock_volume" => self
.market_series(symbol)
.and_then(|series| series.current_volume_moving_average(date, lookback)),
"day_open" | "dayopen" => {
self.market_moving_average(date, symbol, lookback, PriceField::DayOpen)
}
"open" => self.market_moving_average(date, symbol, lookback, PriceField::Open),
"last" | "last_price" => {
self.market_moving_average(date, symbol, lookback, PriceField::Last)
}
other => self.factor_moving_average(date, symbol, other, lookback),
}
}
pub fn market_current_numeric_moving_average_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
field: &str,
lookback: usize,
) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series_by_symbol_id(symbol_id)
.and_then(|series| series.current_moving_average(date, lookback)),
"volume" | "stock_volume" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.current_volume_moving_average(date, lookback)),
"day_open" | "dayopen" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.moving_average(date, lookback, PriceField::DayOpen)),
"open" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.moving_average(date, lookback, PriceField::Open)),
"last" | "last_price" => self
.market_series_by_symbol_id(symbol_id)
.and_then(|series| series.moving_average(date, lookback, PriceField::Last)),
other => self.factor_moving_average(date, symbol, other, lookback),
}
}
pub fn market_latest_back_adjusted_close(&self, date: NaiveDate, symbol: &str) -> Option<f64> {
self.adjusted_close_series(symbol)
.and_then(|series| series.latest_back_adjusted_close(date))
}
pub fn market_decision_numeric_values(
&self,
date: NaiveDate,
symbol: &str,
field: &str,
lookback: usize,
) -> Vec<f64> {
if lookback == 0 {
return Vec::new();
}
let field = normalized_field(field);
match field.as_ref() {
"close" | "prev_close" | "stock_close" | "price" => self
.adjusted_close_series(symbol)
.map(|series| series.values(date, lookback, false))
.unwrap_or_default(),
"volume" | "stock_volume" => self
.market_series(symbol)
.and_then(|series| series.decision_volume_values(date, lookback))
.unwrap_or_default(),
"day_open" | "dayopen" => self
.market_series(symbol)
.map(|series| series.trailing_values(date, lookback, PriceField::DayOpen))
.unwrap_or_default(),
"open" => self
.market_series(symbol)
.map(|series| series.trailing_values(date, lookback, PriceField::Open))
.unwrap_or_default(),
"last" | "last_price" => self
.market_series(symbol)
.map(|series| series.trailing_values(date, lookback, PriceField::Last))
.unwrap_or_default(),
other => self.factor_numeric_values(date, symbol, other, lookback),
}
}
pub fn market_current_numeric_values(
&self,
date: NaiveDate,
symbol: &str,
field: &str,
lookback: usize,
) -> Vec<f64> {
let field = normalized_field(field);
if matches!(
field.as_ref(),
"close" | "prev_close" | "stock_close" | "price"
) {
return self
.adjusted_close_series(symbol)
.map(|series| series.values(date, lookback, true))
.unwrap_or_default();
}
if matches!(field.as_ref(), "volume" | "stock_volume") {
return self
.market_series(symbol)
.and_then(|series| series.current_volume_values(date, lookback))
.unwrap_or_default();
}
self.market_series(symbol)
.map(|series| series.trailing_numeric_values(date, lookback, field.as_ref(), true))
.unwrap_or_default()
}
pub fn factor_numeric_values(
&self,
date: NaiveDate,
symbol: &str,
field: &str,
lookback: usize,
) -> Vec<f64> {
if lookback == 0 {
return Vec::new();
}
self.calendar
.trailing_days(date, lookback)
.into_iter()
.filter_map(|trading_day| self.factor(trading_day, symbol))
.filter_map(|snapshot| factor_numeric_value(snapshot, field))
.collect()
}
pub fn market_moving_average(
&self,
date: NaiveDate,
symbol: &str,
lookback: usize,
field: PriceField,
) -> Option<f64> {
self.market_series(symbol)
.and_then(|series| series.moving_average(date, lookback, field))
}
pub fn benchmark_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
self.benchmark_series_cache.moving_average(date, lookback)
}
pub fn benchmark_decision_close(&self, date: NaiveDate) -> Option<f64> {
self.benchmark_series_cache.decision_close(date)
}
pub fn benchmark_decision_moving_average(
&self,
date: NaiveDate,
lookback: usize,
) -> Option<f64> {
self.benchmark_series_cache
.decision_moving_average(date, lookback)
}
pub fn benchmark_open_moving_average(&self, date: NaiveDate, lookback: usize) -> Option<f64> {
self.benchmark_series_cache
.moving_average_for(date, lookback, PriceField::Open)
}
pub fn benchmark_numeric_values(
&self,
date: NaiveDate,
field: &str,
lookback: usize,
) -> Vec<f64> {
let field = normalize_field(field);
match field.as_str() {
"open" | "day_open" | "dayopen" | "benchmark_open" => self
.benchmark_series_cache
.trailing_values_for(date, lookback, PriceField::Open),
_ => self.benchmark_series_cache.trailing_values(date, lookback),
}
}
pub fn benchmark_decision_numeric_values(
&self,
date: NaiveDate,
field: &str,
lookback: usize,
) -> Vec<f64> {
let field = normalize_field(field);
match field.as_str() {
"open" | "day_open" | "dayopen" | "benchmark_open" => self
.benchmark_series_cache
.trailing_values_for(date, lookback, PriceField::Open),
_ => self
.benchmark_series_cache
.decision_values_for(date, lookback, PriceField::Close),
}
}
pub fn market_open_moving_average(
&self,
date: NaiveDate,
symbol: &str,
lookback: usize,
) -> Option<f64> {
self.market_moving_average(date, symbol, lookback, PriceField::Open)
}
pub fn eligible_universe_on(&self, date: NaiveDate) -> &[EligibleUniverseSnapshot] {
self.eligible_universe_by_date
.get_or_init(|| build_eligible_universe(&self.factor_by_date, &self.market_by_date))
.get(&date)
.map(Vec::as_slice)
.unwrap_or(&[])
}
pub fn fundamental_universe_on(&self, date: NaiveDate) -> Vec<EligibleUniverseSnapshot> {
build_fundamental_universe_for_date(date, &self.factor_by_date, &self.market_by_date)
}
pub fn eligible_universe_on_with_risk_config(
&self,
date: NaiveDate,
risk_config: &FidcRiskControlConfig,
) -> Vec<EligibleUniverseSnapshot> {
build_eligible_universe_for_date(
date,
&self.factor_by_date,
&self.candidate_by_date,
&self.market_by_date,
&self.instruments,
risk_config,
)
}
pub fn require_market(
&self,
date: NaiveDate,
symbol: &str,
) -> Result<&DailyMarketSnapshot, DataSetError> {
self.market(date, symbol)
.ok_or_else(|| DataSetError::MissingSnapshot {
kind: "market",
date,
symbol: symbol.to_string(),
})
}
pub fn require_market_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
) -> Result<&DailyMarketSnapshot, DataSetError> {
self.market_by_symbol_id(date, symbol_id)
.ok_or_else(|| DataSetError::MissingSnapshot {
kind: "market",
date,
symbol: symbol.to_string(),
})
}
pub fn require_candidate(
&self,
date: NaiveDate,
symbol: &str,
) -> Result<&CandidateEligibility, DataSetError> {
self.candidate(date, symbol)
.ok_or_else(|| DataSetError::MissingSnapshot {
kind: "candidate",
date,
symbol: symbol.to_string(),
})
}
pub fn require_candidate_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
) -> Result<&CandidateEligibility, DataSetError> {
self.candidate_by_symbol_id(date, symbol_id)
.ok_or_else(|| DataSetError::MissingSnapshot {
kind: "candidate",
date,
symbol: symbol.to_string(),
})
}
pub fn require_factor(
&self,
date: NaiveDate,
symbol: &str,
) -> Result<&DailyFactorSnapshot, DataSetError> {
self.factor(date, symbol)
.ok_or_else(|| DataSetError::MissingSnapshot {
kind: "factor",
date,
symbol: symbol.to_string(),
})
}
pub fn require_factor_by_symbol_id(
&self,
date: NaiveDate,
symbol_id: u32,
symbol: &str,
) -> Result<&DailyFactorSnapshot, DataSetError> {
self.factor_by_symbol_id(date, symbol_id)
.ok_or_else(|| DataSetError::MissingSnapshot {
kind: "factor",
date,
symbol: symbol.to_string(),
})
}
}
fn normalized_aliases(values: &[String]) -> Vec<String> {
let mut aliases = Vec::new();
for value in values {
let normalized = normalize_field(value);
if !aliases.contains(&normalized) {
aliases.push(normalized);
}
}
aliases
}
fn shares_factor_aliases(share_type: &str) -> Vec<String> {
let field = normalize_field(share_type);
let values = match field.as_str() {
"" | "all" | "total" => vec![
"total_shares",
"shares_total",
"total_share",
"total_share_capital",
"capitalization",
"shares",
],
"float" | "free_float" | "circulating" | "circulation" => vec![
"free_float_shares",
"float_shares",
"circulating_shares",
"circulation_shares",
"float_a_shares",
],
"a" | "a_share" | "a_shares" => vec!["a_shares", "shares_a", "a_share_capital"],
other => {
return normalized_aliases(&[
other.to_string(),
format!("shares_{other}"),
format!("{other}_shares"),
]);
}
};
normalized_aliases(
&values
.iter()
.map(|value| value.to_string())
.collect::<Vec<_>>(),
)
}
fn turnover_rate_factor_aliases(field: &str) -> Vec<String> {
let field = normalize_field(field);
let values = match field.as_str() {
"" | "all" | "rate" | "turnover" | "turnover_rate" | "turnover_ratio" => {
vec!["turnover_rate", "turnover_ratio"]
}
"effective" | "effective_turnover" | "effective_turnover_rate" => {
vec!["effective_turnover_rate", "effective_turnover_ratio"]
}
other => {
return normalized_aliases(&[
other.to_string(),
format!("turnover_rate_{other}"),
format!("{other}_turnover_rate"),
format!("turnover_ratio_{other}"),
format!("{other}_turnover_ratio"),
]);
}
};
normalized_aliases(
&values
.iter()
.map(|value| value.to_string())
.collect::<Vec<_>>(),
)
}
fn stock_connect_factor_aliases(field: &str) -> Vec<String> {
let field = normalize_field(field);
let values = match field.as_str() {
"" | "all" | "connect" | "stock_connect" => {
vec![
"stock_connect",
"stock_connect_all",
"connect_all",
"north_bound",
]
}
"north" | "north_bound" | "northbound" => vec![
"stock_connect_north_bound",
"stock_connect_northbound",
"connect_north_bound",
"north_bound",
"northbound",
],
"south" | "south_bound" | "southbound" => vec![
"stock_connect_south_bound",
"stock_connect_southbound",
"connect_south_bound",
"south_bound",
"southbound",
],
other => {
return normalized_aliases(&[
other.to_string(),
format!("stock_connect_{other}"),
format!("connect_{other}"),
]);
}
};
normalized_aliases(
&values
.iter()
.map(|value| value.to_string())
.collect::<Vec<_>>(),
)
}
fn prefixed_factor_aliases(prefix: &str, field: &str) -> Vec<String> {
let prefix = normalize_field(prefix);
let field = normalize_field(field);
let plural_prefix = format!("{prefix}s");
normalized_aliases(&[
format!("{prefix}_{field}"),
format!("{plural_prefix}_{field}"),
field.clone(),
])
}
fn industry_factor_aliases(source: &str, level: usize) -> Vec<String> {
let source = normalize_field(source);
normalized_aliases(&[
format!("industry_{source}_l{level}"),
format!("industry_{source}_{level}"),
format!("{source}_industry_l{level}"),
format!("{source}_industry_{level}"),
format!("industry_l{level}"),
format!("industry_{level}"),
"industry_code".to_string(),
])
}
fn industry_name_factor_aliases(source: &str, level: usize) -> Vec<String> {
let source = normalize_field(source);
normalized_aliases(&[
format!("industry_{source}_l{level}_name"),
format!("industry_{source}_{level}_name"),
format!("industry_{source}_name_l{level}"),
format!("{source}_industry_l{level}_name"),
format!("{source}_industry_{level}_name"),
format!("{source}_industry_name_l{level}"),
format!("industry_l{level}_name"),
format!("industry_{level}_name"),
"industry_name".to_string(),
])
}
fn factor_numeric_value(snapshot: &DailyFactorSnapshot, field: &str) -> Option<f64> {
let field = normalized_field(field);
match field.as_ref() {
"market_cap" | "market_cap_bn" => Some(snapshot.market_cap_bn),
"free_float_cap" | "free_float_market_cap" | "free_float_cap_bn" => {
Some(snapshot.free_float_cap_bn)
}
"free_float_cap_or_market_cap" => Some(
(snapshot.free_float_cap_bn.is_finite() && snapshot.free_float_cap_bn > 0.0)
.then_some(snapshot.free_float_cap_bn)
.unwrap_or(snapshot.market_cap_bn),
),
"pe_ttm" => Some(snapshot.pe_ttm),
"turnover_ratio" => snapshot.turnover_ratio,
"effective_turnover_ratio" => snapshot.effective_turnover_ratio,
"ths_market_value_stock" | "ths_market_value_stock_bn" => snapshot
.extra_factors
.get(field.as_ref())
.copied()
.or(Some(snapshot.market_cap_bn)),
"ths_current_mv_stock" | "ths_current_mv_stock_bn" => snapshot
.extra_factors
.get(field.as_ref())
.copied()
.or(Some(snapshot.free_float_cap_bn)),
"ths_turnover_ratio_stock" => snapshot
.extra_factors
.get(field.as_ref())
.copied()
.or(snapshot.turnover_ratio),
"ths_vaild_turnover_stock" | "ths_valid_turnover_stock" => snapshot
.extra_factors
.get(field.as_ref())
.copied()
.or(snapshot.effective_turnover_ratio),
other => snapshot.extra_factors.get(other).copied(),
}
}
fn intraday_quote_numeric_value(snapshot: &IntradayExecutionQuote, field: &str) -> Option<f64> {
match normalized_field(field).as_ref() {
"last" | "last_price" | "close" | "price" => Some(snapshot.last_price),
"bid1" => Some(snapshot.bid1),
"ask1" => Some(snapshot.ask1),
"bid1_volume" => Some(snapshot.bid1_volume as f64),
"ask1_volume" => Some(snapshot.ask1_volume as f64),
"volume" | "volume_delta" => Some(snapshot.volume_delta as f64),
"amount" | "amount_delta" | "total_turnover" => Some(snapshot.amount_delta),
_ => None,
}
}
fn intraday_quote_visible(
quote: &IntradayExecutionQuote,
date: NaiveDate,
active_datetime: Option<NaiveDateTime>,
include_now: bool,
) -> bool {
if quote.date < date {
return true;
}
if quote.date > date {
return false;
}
let Some(active_datetime) = active_datetime.filter(|value| value.date() == date) else {
return include_now;
};
if include_now {
quote.timestamp <= active_datetime
} else {
quote.timestamp < active_datetime
}
}
fn daily_market_price_bar(snapshot: &DailyMarketSnapshot) -> PriceBar {
PriceBar {
date: snapshot.date,
timestamp: snapshot.timestamp.clone(),
symbol: snapshot.symbol.clone(),
frequency: "1d".to_string(),
open: snapshot.open,
high: snapshot.high,
low: snapshot.low,
close: snapshot.close,
last_price: snapshot.last_price,
volume: snapshot.volume,
amount: 0.0,
bid1: snapshot.bid1,
ask1: snapshot.ask1,
bid1_volume: snapshot.bid1_volume,
ask1_volume: snapshot.ask1_volume,
}
}
fn intraday_quote_price_bar(snapshot: &IntradayExecutionQuote) -> PriceBar {
PriceBar {
date: snapshot.date,
timestamp: Some(snapshot.timestamp.format("%Y-%m-%d %H:%M:%S").to_string()),
symbol: snapshot.symbol.clone(),
frequency: "1m".to_string(),
open: snapshot.last_price,
high: snapshot.last_price,
low: snapshot.last_price,
close: snapshot.last_price,
last_price: snapshot.last_price,
volume: snapshot.volume_delta,
amount: snapshot.amount_delta,
bid1: snapshot.bid1,
ask1: snapshot.ask1,
bid1_volume: snapshot.bid1_volume,
ask1_volume: snapshot.ask1_volume,
}
}
fn normalize_field(field: &str) -> String {
normalized_field(field).into_owned()
}
fn normalized_field(field: &str) -> Cow<'_, str> {
let trimmed = field.trim().trim_matches('"').trim_matches('\'');
if trimmed.bytes().all(|byte| !byte.is_ascii_uppercase()) {
Cow::Borrowed(trimmed)
} else {
Cow::Owned(trimmed.to_ascii_lowercase())
}
}
fn normalize_factor_snapshots(factors: Vec<DailyFactorSnapshot>) -> Vec<DailyFactorSnapshot> {
factors
.into_iter()
.map(|mut snapshot| {
let already_normalized = snapshot.extra_factors.iter().all(|(field, value)| {
let trimmed = field.as_ref().trim().trim_matches('"').trim_matches('\'');
!trimmed.is_empty()
&& trimmed == field.as_ref()
&& trimmed.bytes().all(|byte| !byte.is_ascii_uppercase())
&& value.is_finite()
});
if already_normalized {
return snapshot;
}
snapshot.extra_factors = snapshot
.extra_factors
.into_iter()
.filter_map(|(field, value)| {
let trimmed = field.as_ref().trim().trim_matches('"').trim_matches('\'');
if trimmed.is_empty() || !value.is_finite() {
None
} else if trimmed == field.as_ref()
&& trimmed.bytes().all(|byte| !byte.is_ascii_uppercase())
{
Some((field, value))
} else {
Some((Cow::Owned(trimmed.to_ascii_lowercase()), value))
}
})
.collect();
snapshot
})
.collect()
}
fn normalize_history_frequency(frequency: &str) -> Option<String> {
let normalized = normalize_field(frequency);
match normalized.as_str() {
"1d" | "d" | "day" | "daily" => Some("1d".to_string()),
"1m" | "m" | "minute" | "min" => Some("1m".to_string()),
_ => None,
}
}
fn group_by_date<T, F>(rows: Vec<T>, mut date_of: F) -> BTreeMap<NaiveDate, Vec<T>>
where
F: FnMut(&T) -> NaiveDate,
{
let mut grouped = BTreeMap::<NaiveDate, Vec<T>>::new();
for row in rows {
grouped.entry(date_of(&row)).or_default().push(row);
}
grouped
}
fn sort_groups_by_symbol<T, F>(groups: &mut BTreeMap<NaiveDate, Vec<T>>, symbol_of: F)
where
F: Fn(&T) -> &str + Copy,
{
for rows in groups.values_mut() {
rows.sort_by(|left, right| symbol_of(left).cmp(symbol_of(right)));
}
}
fn build_symbol_id_index(
instruments: &HashMap<String, Instrument>,
market_by_date: &BTreeMap<NaiveDate, Vec<DailyMarketSnapshot>>,
factor_by_date: &BTreeMap<NaiveDate, Vec<DailyFactorSnapshot>>,
candidate_by_date: &BTreeMap<NaiveDate, Vec<CandidateEligibility>>,
) -> AHashMap<String, u32> {
let mut symbols = instruments.keys().cloned().collect::<HashSet<_>>();
for rows in market_by_date.values() {
for row in rows {
if !symbols.contains(row.symbol.as_str()) {
symbols.insert(row.symbol.clone());
}
}
}
for rows in factor_by_date.values() {
for row in rows {
if !symbols.contains(row.symbol.as_str()) {
symbols.insert(row.symbol.clone());
}
}
}
for rows in candidate_by_date.values() {
for row in rows {
if !symbols.contains(row.symbol.as_str()) {
symbols.insert(row.symbol.clone());
}
}
}
let mut symbols = symbols.into_iter().collect::<Vec<_>>();
symbols.sort_unstable();
symbols
.into_iter()
.enumerate()
.map(|(index, symbol)| {
(
symbol,
u32::try_from(index).expect("FIDC symbol index exceeds u32 capacity"),
)
})
.collect()
}
fn build_group_symbol_ids<T, F>(
groups: &BTreeMap<NaiveDate, Vec<T>>,
symbol_id_by_code: &AHashMap<String, u32>,
symbol_of: F,
) -> BTreeMap<NaiveDate, Vec<u32>>
where
F: Fn(&T) -> &str + Copy,
{
groups
.iter()
.map(|(date, rows)| {
let symbol_ids = rows
.iter()
.map(|row| {
*symbol_id_by_code
.get(symbol_of(row))
.expect("snapshot symbol missing from FIDC symbol index")
})
.collect::<Vec<_>>();
debug_assert!(symbol_ids.windows(2).all(|window| window[0] < window[1]));
(*date, symbol_ids)
})
.collect()
}
fn find_by_symbol_id<'a, T>(rows: &'a [T], symbol_ids: &[u32], symbol_id: u32) -> Option<&'a T> {
find_by_symbol_id_with_preferred_index(rows, symbol_ids, symbol_id, None)
}
fn symbol_id_index(rows_len: usize, symbol_ids: &[u32], symbol_id: u32) -> Option<usize> {
if rows_len != symbol_ids.len() {
return None;
}
symbol_ids.binary_search(&symbol_id).ok()
}
fn find_by_symbol_id_with_preferred_index<'a, T>(
rows: &'a [T],
symbol_ids: &[u32],
symbol_id: u32,
preferred_index: Option<usize>,
) -> Option<&'a T> {
if rows.len() != symbol_ids.len() {
return None;
}
if let Some(index) = preferred_index
&& symbol_ids.get(index).copied() == Some(symbol_id)
{
return rows.get(index);
}
symbol_ids
.binary_search(&symbol_id)
.ok()
.and_then(|index| rows.get(index))
}
fn find_by_symbol<'a, T, F>(rows: &'a [T], symbol: &str, symbol_of: F) -> Option<&'a T>
where
F: Fn(&T) -> &str,
{
rows.binary_search_by(|row| symbol_of(row).cmp(symbol))
.ok()
.map(|index| &rows[index])
}
fn collect_benchmark_code(benchmarks: &[BenchmarkSnapshot]) -> Result<String, DataSetError> {
let mut codes = benchmarks
.iter()
.map(|row| row.benchmark.clone())
.collect::<Vec<_>>();
codes.sort_unstable();
codes.dedup();
if codes.len() == 1 {
Ok(codes.remove(0))
} else {
Err(DataSetError::MultipleBenchmarks)
}
}
fn prefix_sums(values: &[f64]) -> Vec<f64> {
let mut prefix = Vec::with_capacity(values.len() + 1);
prefix.push(0.0);
for value in values {
let next = prefix.last().copied().unwrap_or_default() + *value;
prefix.push(next);
}
prefix
}
fn normalize_rolling_factor(value: f64, decimals: i32) -> f64 {
let scale = 10_f64.powi(decimals);
(value * scale).round() / scale
}
mod optional_date_format {
use chrono::NaiveDate;
use serde::{self, Deserialize, Deserializer, Serializer};
const FORMAT: &str = "%Y-%m-%d";
pub fn serialize<S>(date: &Option<NaiveDate>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match date {
Some(date) => serializer.serialize_some(&date.format(FORMAT).to_string()),
None => serializer.serialize_none(),
}
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<NaiveDate>, D::Error>
where
D: Deserializer<'de>,
{
let text = Option::<String>::deserialize(deserializer)?;
match text
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
Some(text) => NaiveDate::parse_from_str(text, FORMAT)
.map(Some)
.map_err(serde::de::Error::custom),
None => Ok(None),
}
}
}
fn build_futures_params_index(
rows: Vec<FuturesTradingParameter>,
) -> HashMap<String, Vec<FuturesTradingParameter>> {
let mut grouped = HashMap::<String, Vec<FuturesTradingParameter>>::new();
for row in rows {
grouped.entry(row.symbol.clone()).or_default().push(row);
}
for rows in grouped.values_mut() {
rows.sort_by_key(|row| row.effective_date);
}
grouped
}
fn build_execution_quote_index(
execution_quotes: Vec<IntradayExecutionQuote>,
) -> HashMap<NaiveDate, HashMap<String, Vec<IntradayExecutionQuote>>> {
let mut grouped = HashMap::<NaiveDate, HashMap<String, Vec<IntradayExecutionQuote>>>::new();
for quote in execution_quotes {
grouped
.entry(quote.date)
.or_default()
.entry(quote.symbol.clone())
.or_default()
.push(quote);
}
for rows_by_symbol in grouped.values_mut() {
for quotes in rows_by_symbol.values_mut() {
quotes.sort_by_key(|quote| quote.timestamp);
}
}
grouped
}
fn build_order_book_depth_index(
order_book_depth: Vec<IntradayOrderBookDepthLevel>,
) -> HashMap<(NaiveDate, String), Vec<IntradayOrderBookDepthLevel>> {
let mut grouped = HashMap::<(NaiveDate, String), Vec<IntradayOrderBookDepthLevel>>::new();
for level in order_book_depth {
grouped
.entry((level.date, level.symbol.clone()))
.or_default()
.push(level);
}
for levels in grouped.values_mut() {
levels.sort_by(|left, right| {
left.timestamp
.cmp(&right.timestamp)
.then(left.level.cmp(&right.level))
});
}
grouped
}
fn build_eligible_universe(
factor_by_date: &BTreeMap<NaiveDate, Vec<DailyFactorSnapshot>>,
market_by_date: &BTreeMap<NaiveDate, Vec<DailyMarketSnapshot>>,
) -> BTreeMap<NaiveDate, Vec<EligibleUniverseSnapshot>> {
let mut per_date = BTreeMap::<NaiveDate, Vec<EligibleUniverseSnapshot>>::new();
for date in factor_by_date.keys() {
let rows = build_fundamental_universe_for_date(*date, factor_by_date, market_by_date);
per_date.insert(*date, rows);
}
per_date
}
fn build_fundamental_universe_for_date(
date: NaiveDate,
factor_by_date: &BTreeMap<NaiveDate, Vec<DailyFactorSnapshot>>,
market_by_date: &BTreeMap<NaiveDate, Vec<DailyMarketSnapshot>>,
) -> Vec<EligibleUniverseSnapshot> {
let mut rows = Vec::new();
let Some(factors) = factor_by_date.get(&date) else {
return rows;
};
for factor in factors {
if market_by_date
.get(&date)
.and_then(|rows| find_by_symbol(rows, &factor.symbol, |row| row.symbol.as_str()))
.is_none()
{
continue;
}
let market_cap_bn = decision_market_cap_bn(factor);
if market_cap_bn <= 0.0 || !market_cap_bn.is_finite() {
continue;
}
rows.push(EligibleUniverseSnapshot {
symbol: factor.symbol.clone(),
market_cap_bn,
free_float_cap_bn: decision_free_float_cap_bn(factor),
});
}
rows.sort_by(|left, right| {
left.market_cap_bn
.partial_cmp(&right.market_cap_bn)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| left.symbol.cmp(&right.symbol))
});
rows
}
fn build_eligible_universe_for_date(
date: NaiveDate,
factor_by_date: &BTreeMap<NaiveDate, Vec<DailyFactorSnapshot>>,
candidate_by_date: &BTreeMap<NaiveDate, Vec<CandidateEligibility>>,
market_by_date: &BTreeMap<NaiveDate, Vec<DailyMarketSnapshot>>,
instruments: &HashMap<String, Instrument>,
risk_config: &FidcRiskControlConfig,
) -> Vec<EligibleUniverseSnapshot> {
factor_by_date
.get(&date)
.map(|factors| {
build_eligible_universe_for_date_from_factors(
date,
factors,
candidate_by_date,
market_by_date,
instruments,
risk_config,
)
})
.unwrap_or_default()
}
fn build_eligible_universe_for_date_from_factors(
date: NaiveDate,
factors: &[DailyFactorSnapshot],
candidate_by_date: &BTreeMap<NaiveDate, Vec<CandidateEligibility>>,
market_by_date: &BTreeMap<NaiveDate, Vec<DailyMarketSnapshot>>,
instruments: &HashMap<String, Instrument>,
risk_config: &FidcRiskControlConfig,
) -> Vec<EligibleUniverseSnapshot> {
let mut rows = Vec::new();
for factor in factors {
if factor.market_cap_bn <= 0.0 || !factor.market_cap_bn.is_finite() {
continue;
}
let synthetic_candidate;
let candidate = if let Some(candidate) = candidate_by_date
.get(&date)
.and_then(|rows| find_by_symbol(rows, &factor.symbol, |row| row.symbol.as_str()))
{
candidate
} else {
synthetic_candidate = missing_candidate_risk_state(date, &factor.symbol);
&synthetic_candidate
};
let Some(market) = market_by_date
.get(&date)
.and_then(|rows| find_by_symbol(rows, &factor.symbol, |row| row.symbol.as_str()))
else {
continue;
};
if ChinaAShareRiskControl::selection_rejection_reason_with_config(
date,
candidate,
market,
instruments.get(&factor.symbol),
risk_config,
)
.is_some()
{
continue;
}
let market_cap_bn = decision_market_cap_bn(factor);
if market_cap_bn <= 0.0 || !market_cap_bn.is_finite() {
continue;
}
let free_float_cap_bn = decision_free_float_cap_bn(factor);
rows.push(EligibleUniverseSnapshot {
symbol: factor.symbol.clone(),
market_cap_bn,
free_float_cap_bn,
});
}
rows.sort_by(|left, right| {
left.market_cap_bn
.partial_cmp(&right.market_cap_bn)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| left.symbol.cmp(&right.symbol))
});
rows
}
pub(crate) fn missing_candidate_risk_state(date: NaiveDate, symbol: &str) -> CandidateEligibility {
CandidateEligibility {
date,
symbol: symbol.to_string(),
is_st: false,
is_star_st: false,
is_new_listing: false,
is_paused: false,
allow_buy: true,
allow_sell: true,
is_kcb: false,
is_one_yuan: false,
risk_level_code: Some(
"missing_risk_state:is_st,is_star_st,is_paused,listed_days,is_kcb,is_one_yuan"
.to_string(),
),
}
}
#[cfg(test)]
fn instrument_passes_baseline_selection(instrument: Option<&Instrument>, date: NaiveDate) -> bool {
ChinaAShareRiskControl::instrument_rejection_reason(instrument, date).is_none()
}
#[cfg(test)]
mod tests {
use super::*;
fn market_row(date: &str, prev_close: f64, volume: u64) -> DailyMarketSnapshot {
DailyMarketSnapshot {
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
symbol: "000001.SZ".to_string(),
timestamp: None,
day_open: prev_close,
open: prev_close,
high: prev_close,
low: prev_close,
close: prev_close,
last_price: prev_close,
bid1: prev_close,
ask1: prev_close,
prev_close,
volume,
minute_volume: 0,
bid1_volume: 0,
ask1_volume: 0,
trading_phase: None,
paused: false,
upper_limit: prev_close * 1.1,
lower_limit: prev_close * 0.9,
price_tick: 0.01,
}
}
fn benchmark_row(date: &str, close: f64) -> BenchmarkSnapshot {
BenchmarkSnapshot {
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
benchmark: "000852.SH".to_string(),
open: close,
close,
prev_close: close - 1.0,
volume: 1_000_000,
}
}
#[test]
fn dataset_clone_shares_immutable_base_and_isolates_execution_quotes() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let data = DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "平安银行".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
}],
vec![market_row("2025-01-02", 10.0, 1_000_000)],
Vec::new(),
Vec::new(),
vec![benchmark_row("2025-01-02", 12.0)],
)
.unwrap();
let mut run_data = data.clone();
assert!(Arc::ptr_eq(&data.instruments, &run_data.instruments));
assert!(Arc::ptr_eq(&data.market_by_date, &run_data.market_by_date));
assert!(Arc::ptr_eq(&data.factor_by_date, &run_data.factor_by_date));
assert!(Arc::ptr_eq(
&data.candidate_by_date,
&run_data.candidate_by_date
));
assert!(Arc::ptr_eq(
&data.benchmark_by_date,
&run_data.benchmark_by_date
));
assert!(Arc::ptr_eq(
&data.execution_quotes_by_date,
&run_data.execution_quotes_by_date
));
assert!(Arc::ptr_eq(
&data.execution_quote_dates,
&run_data.execution_quote_dates
));
run_data.add_execution_quotes(vec![IntradayExecutionQuote {
date,
timestamp: NaiveDateTime::parse_from_str("2025-01-02 10:18:00", "%Y-%m-%d %H:%M:%S")
.unwrap(),
symbol: "000001.SZ".to_string(),
last_price: 10.01,
bid1: 10.0,
ask1: 10.01,
bid1_volume: 10_000,
ask1_volume: 10_000,
volume_delta: 10_000,
amount_delta: 100_100.0,
trading_phase: Some("continuous".to_string()),
}]);
assert_eq!(data.execution_quote_count(), 0);
assert_eq!(run_data.execution_quote_count(), 1);
assert!(!Arc::ptr_eq(
&data.execution_quotes_by_date,
&run_data.execution_quotes_by_date
));
assert!(!Arc::ptr_eq(
&data.execution_quote_dates,
&run_data.execution_quote_dates
));
}
#[test]
fn combined_symbol_snapshot_lookup_uses_alignment_and_falls_back_for_sparse_rows() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let instrument = |symbol: &str| Instrument {
symbol: symbol.to_string(),
name: symbol.to_string(),
board: symbol
.rsplit_once('.')
.map(|(_, value)| value)
.unwrap_or("")
.to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
};
let market = |symbol: &str, close: f64| {
let mut row = market_row("2025-01-02", close, 1_000_000);
row.symbol = symbol.to_string();
row
};
let factor = |symbol: &str, market_cap_bn: f64| DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
market_cap_bn,
free_float_cap_bn: market_cap_bn,
pe_ttm: 0.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: NumericFactorMap::new(),
};
let candidate = |symbol: &str| CandidateEligibility {
date,
symbol: symbol.to_string(),
is_st: false,
is_star_st: false,
is_new_listing: false,
is_paused: false,
allow_buy: true,
allow_sell: true,
is_kcb: false,
is_one_yuan: false,
risk_level_code: None,
};
let data = DataSet::from_components(
vec![
instrument("000001.SZ"),
instrument("000300.SH"),
instrument("600000.SH"),
],
vec![
market("000001.SZ", 10.0),
market("000300.SH", 20.0),
market("600000.SH", 12.0),
],
vec![factor("000001.SZ", 100.0), factor("600000.SH", 120.0)],
vec![candidate("000001.SZ"), candidate("600000.SH")],
vec![benchmark_row("2025-01-02", 20.0)],
)
.unwrap();
for symbol in ["000001.SZ", "600000.SH"] {
let symbol_id = data.symbol_id(symbol).unwrap();
let combined = data.symbol_snapshots_by_id(date, symbol_id);
assert_eq!(
combined.market.map(|row| row.symbol.as_str()),
data.market_by_symbol_id(date, symbol_id)
.map(|row| row.symbol.as_str())
);
assert_eq!(
combined.factor.map(|row| row.symbol.as_str()),
data.factor_by_symbol_id(date, symbol_id)
.map(|row| row.symbol.as_str())
);
assert_eq!(
combined.candidate.map(|row| row.symbol.as_str()),
data.candidate_by_symbol_id(date, symbol_id)
.map(|row| row.symbol.as_str())
);
}
let signal_id = data.symbol_id("000300.SH").unwrap();
let signal = data.symbol_snapshots_by_id(date, signal_id);
assert_eq!(signal.market.map(|row| row.symbol.as_str()), Some("000300.SH"));
assert!(signal.factor.is_none());
assert!(signal.candidate.is_none());
}
#[test]
fn decision_rolling_cursor_matches_existing_close_and_volume_contract() {
let dates = ["2025-01-02", "2025-01-03", "2025-01-06", "2025-01-07"];
let market = dates
.iter()
.enumerate()
.map(|(index, date)| market_row(date, 10.0 + index as f64, 1_000 + index as u64 * 100))
.collect::<Vec<_>>();
let factors = dates
.iter()
.map(|date| DailyFactorSnapshot {
date: NaiveDate::parse_from_str(date, "%Y-%m-%d").unwrap(),
symbol: "000001.SZ".to_string(),
market_cap_bn: 100.0,
free_float_cap_bn: 80.0,
pe_ttm: 0.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: NumericFactorMap::from([(
Cow::Borrowed(BACKWARD_ADJUSTMENT_FACTOR_FIELD),
1.0,
)]),
})
.collect::<Vec<_>>();
let benchmarks = dates
.iter()
.enumerate()
.map(|(index, date)| benchmark_row(date, 20.0 + index as f64))
.collect::<Vec<_>>();
let data = DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "000001.SZ".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
}],
market,
factors,
Vec::new(),
benchmarks,
)
.unwrap();
let symbol_id = data.symbol_id("000001.SZ").unwrap();
let date = NaiveDate::parse_from_str("2025-01-07", "%Y-%m-%d").unwrap();
let cursor = data.market_decision_rolling_cursor_by_symbol_id(date, symbol_id);
for lookback in [1, 2, 3] {
assert_eq!(
cursor.moving_average("close", lookback),
data.market_decision_numeric_moving_average_by_symbol_id(
date,
symbol_id,
"000001.SZ",
"close",
lookback,
)
);
assert_eq!(
cursor.moving_average("volume", lookback),
data.market_decision_numeric_moving_average_by_symbol_id(
date,
symbol_id,
"000001.SZ",
"volume",
lookback,
)
);
}
}
#[test]
fn additional_terminal_calendar_dates_are_isolated_from_shared_market_data() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let next_date = NaiveDate::parse_from_str("2025-01-03", "%Y-%m-%d").unwrap();
let data = DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "平安银行".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
}],
vec![market_row("2025-01-02", 10.0, 1_000_000)],
Vec::new(),
Vec::new(),
vec![benchmark_row("2025-01-02", 12.0)],
)
.unwrap();
let run_data = data
.clone()
.with_additional_trading_dates([next_date, next_date]);
assert_eq!(data.next_trading_date(date, 1), None);
assert_eq!(run_data.next_trading_date(date, 1), Some(next_date));
assert!(run_data.market(next_date, "000001.SZ").is_none());
assert!(Arc::ptr_eq(&data.market_by_date, &run_data.market_by_date));
}
#[test]
fn execution_quotes_use_stable_k_way_merge_and_release_by_date() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let data = DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "平安银行".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: None,
delisted_at: None,
status: "active".to_string(),
}],
vec![market_row("2025-01-02", 10.0, 1_000_000)],
Vec::new(),
Vec::new(),
vec![benchmark_row("2025-01-02", 12.0)],
)
.unwrap();
let quote = |symbol: &str, time: &str| IntradayExecutionQuote {
date,
timestamp: NaiveDateTime::parse_from_str(
&format!("2025-01-02 {time}"),
"%Y-%m-%d %H:%M:%S",
)
.unwrap(),
symbol: symbol.to_string(),
last_price: 10.0,
bid1: 0.0,
ask1: 0.0,
bid1_volume: 0,
ask1_volume: 0,
volume_delta: 100,
amount_delta: 1_000.0,
trading_phase: Some("continuous".to_string()),
};
let mut run_data = data.clone();
assert_eq!(
run_data.add_execution_quotes(vec![
quote("000002.SZ", "09:31:00"),
quote("000001.SZ", "09:31:00"),
quote("000002.SZ", "09:30:00"),
quote("000001.SZ", "09:30:00"),
]),
4
);
let mut conflicting = quote("000001.SZ", "09:31:00");
conflicting.last_price = 99.0;
assert_eq!(
run_data.add_execution_quotes(vec![
conflicting,
quote("000001.SZ", "09:32:00"),
quote("000001.SZ", "09:32:00"),
]),
1
);
let merged = run_data.execution_quotes_on_date(date);
let keys = merged
.iter()
.map(|row| (row.timestamp.time().to_string(), row.symbol.clone()))
.collect::<Vec<_>>();
assert_eq!(
keys,
vec![
("09:30:00".to_string(), "000001.SZ".to_string()),
("09:30:00".to_string(), "000002.SZ".to_string()),
("09:31:00".to_string(), "000001.SZ".to_string()),
("09:31:00".to_string(), "000002.SZ".to_string()),
("09:32:00".to_string(), "000001.SZ".to_string()),
]
);
let streamed_keys = run_data
.execution_quotes_iter_on_date_for_symbols(date, None)
.map(|row| (row.timestamp.time().to_string(), row.symbol.clone()))
.collect::<Vec<_>>();
assert_eq!(streamed_keys, keys);
assert_eq!(merged[2].last_price, 10.0);
let allowed_symbols = BTreeSet::from(["000001.SZ".to_string()]);
let filtered = run_data.execution_quotes_on_date_for_symbols(date, Some(&allowed_symbols));
assert_eq!(
filtered
.iter()
.map(|row| (row.timestamp.time().to_string(), row.symbol.clone()))
.collect::<Vec<_>>(),
vec![
("09:30:00".to_string(), "000001.SZ".to_string()),
("09:31:00".to_string(), "000001.SZ".to_string()),
("09:32:00".to_string(), "000001.SZ".to_string()),
]
);
assert_eq!(run_data.remove_execution_quotes_on_date(date), 5);
assert_eq!(run_data.execution_quote_count(), 0);
}
#[test]
fn baseline_selection_uses_structured_instrument_dates_and_status_only() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let instrument = |name: &str, status: &str, delisted_at: Option<NaiveDate>| Instrument {
symbol: "000001.SZ".to_string(),
name: name.to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: Some(NaiveDate::parse_from_str("2020-01-01", "%Y-%m-%d").unwrap()),
delisted_at,
status: status.to_string(),
};
assert!(instrument_passes_baseline_selection(
Some(&instrument("Short History Stock", "active", None)),
date
));
assert!(instrument_passes_baseline_selection(
Some(&instrument("*ST测试", "active", None)),
date
));
assert!(instrument_passes_baseline_selection(
Some(&instrument("ST测试", "active", None)),
date
));
assert!(instrument_passes_baseline_selection(
Some(&instrument("退市测试", "active", None)),
date
));
assert!(!instrument_passes_baseline_selection(
Some(&instrument("正常名称", "delisted", None)),
date
));
assert!(instrument_passes_baseline_selection(
Some(&instrument(
"正常名称",
"delisted",
Some(NaiveDate::parse_from_str("2025-04-30", "%Y-%m-%d").unwrap()),
)),
date
));
assert!(!instrument_passes_baseline_selection(
Some(&instrument(
"正常名称",
"active",
Some(NaiveDate::parse_from_str("2025-01-01", "%Y-%m-%d").unwrap()),
)),
date
));
}
#[test]
fn factor_numeric_value_normalizes_fields_without_changing_aliases() {
let snapshot = DailyFactorSnapshot {
date: NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap(),
symbol: "000001.SZ".to_string(),
market_cap_bn: 12.5,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: BTreeMap::from([("custom_factor".into(), 3.5)]),
};
assert_eq!(factor_numeric_value(&snapshot, " MARKET_CAP "), Some(12.5));
assert_eq!(factor_numeric_value(&snapshot, "CUSTOM_FACTOR"), Some(3.5));
}
#[test]
fn factor_snapshot_normalization_moves_clean_maps_and_repairs_dirty_maps() {
let date = NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap();
let clean = normalize_factor_snapshots(vec![DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
market_cap_bn: 1.0,
free_float_cap_bn: 1.0,
pe_ttm: 1.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: BTreeMap::from([(Cow::Borrowed("amount"), 10.0)]),
}]);
assert!(matches!(
clean[0].extra_factors.keys().next(),
Some(Cow::Borrowed("amount"))
));
let dirty = normalize_factor_snapshots(vec![DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
market_cap_bn: 1.0,
free_float_cap_bn: 1.0,
pe_ttm: 1.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: BTreeMap::from([
(Cow::Owned(" CUSTOM_FACTOR ".to_string()), 2.0),
(Cow::Borrowed("bad_nan"), f64::NAN),
]),
}]);
assert_eq!(dirty[0].extra_factors.get("custom_factor"), Some(&2.0));
assert!(!dirty[0].extra_factors.contains_key("bad_nan"));
}
#[test]
fn symbol_price_series_test_constructor_sorts_unsorted_rows() {
let series = SymbolPriceSeries::new(
"000001.SZ".to_string(),
&[
market_row("2025-01-06", 12.0, 300),
market_row("2025-01-02", 10.0, 100),
market_row("2025-01-03", 11.0, 200),
],
);
assert!(series.dates.windows(2).all(|window| window[0] < window[1]));
assert_eq!(series.closes, vec![10.0, 11.0, 12.0]);
}
#[test]
fn decision_volume_average_uses_previous_completed_days_only() {
let series = SymbolPriceSeries::new(
"000001.SZ".to_string(),
&[
market_row("2025-01-02", 10.0, 100),
market_row("2025-01-03", 11.0, 200),
market_row("2025-01-06", 12.0, 10_000),
],
);
assert_eq!(
series.decision_close_moving_average(
NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap(),
2
),
Some(11.5)
);
assert_eq!(
series.decision_volume_moving_average(
NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap(),
2
),
Some(150.0)
);
assert_eq!(
series.decision_volume_moving_average(
NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap(),
3
),
None
);
}
fn volume_contract_data(availability: Option<[f64; 3]>) -> DataSet {
let dates = [
NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-03", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap(),
];
let volumes = [100_u64, 0, 300];
DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "000001.SZ".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: Some(dates[0]),
delisted_at: None,
status: "active".to_string(),
}],
dates
.iter()
.zip(volumes)
.map(|(date, volume)| {
market_row(&date.format("%Y-%m-%d").to_string(), 10.0, volume)
})
.collect(),
dates
.iter()
.enumerate()
.map(|(index, date)| {
let mut extra_factors = BTreeMap::new();
if let Some(values) = availability {
extra_factors.insert("source_daily_volume_available".into(), values[index]);
if values[index] >= 0.5 {
extra_factors.insert("daily_volume".into(), volumes[index] as f64);
}
}
DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors,
}
})
.collect(),
Vec::new(),
dates
.iter()
.map(|date| BenchmarkSnapshot {
date: *date,
benchmark: "000852.SH".to_string(),
open: 100.0,
close: 100.0,
prev_close: 100.0,
volume: 1_000_000,
})
.collect(),
)
.expect("volume contract dataset")
}
#[test]
fn source_volume_contract_rejects_windows_containing_missing_values() {
let data = volume_contract_data(Some([1.0, 0.0, 1.0]));
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
assert_eq!(
data.market_current_numeric_moving_average(date, "000001.SZ", "volume", 3),
None
);
assert!(
data.market_current_numeric_values(date, "000001.SZ", "volume", 3)
.is_empty()
);
assert_eq!(
data.market_decision_numeric_moving_average(date, "000001.SZ", "volume", 2),
None
);
assert!(
data.market_decision_numeric_values(date, "000001.SZ", "volume", 2)
.is_empty()
);
}
#[test]
fn volume_rolling_ignores_zero_volume_rows_for_source_and_legacy_data() {
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
for data in [
volume_contract_data(Some([1.0, 1.0, 1.0])),
volume_contract_data(None),
] {
let symbol_id = data.symbol_id("000001.SZ").expect("symbol id");
assert!(std::ptr::eq(
data.market_by_symbol_id(date, symbol_id)
.expect("market by id"),
data.market(date, "000001.SZ").expect("market by code"),
));
assert!(std::ptr::eq(
data.factor_by_symbol_id(date, symbol_id)
.expect("factor by id"),
data.factor(date, "000001.SZ").expect("factor by code"),
));
assert_eq!(
data.market_current_numeric_moving_average(date, "000001.SZ", "volume", 2),
Some(200.0)
);
assert_eq!(
data.market_current_numeric_moving_average_by_symbol_id(
date,
symbol_id,
"000001.SZ",
"volume",
2,
),
Some(200.0)
);
assert_eq!(
data.market_current_numeric_values(date, "000001.SZ", "volume", 2),
vec![100.0, 300.0]
);
assert_eq!(
data.market_current_numeric_moving_average(date, "000001.SZ", "volume", 3),
None
);
assert_eq!(
data.market_decision_numeric_moving_average(date, "000001.SZ", "volume", 1),
Some(100.0)
);
assert_eq!(
data.market_decision_numeric_moving_average_by_symbol_id(
date,
symbol_id,
"000001.SZ",
"volume",
1,
),
Some(100.0)
);
assert_eq!(
data.market_decision_numeric_values(date, "000001.SZ", "volume", 1),
vec![100.0]
);
}
}
#[test]
fn decision_close_average_ignores_current_day_close() {
let mut current = market_row("2025-01-06", 12.0, 10_000);
current.close = 9_999.0;
current.last_price = 9_999.0;
let series = SymbolPriceSeries::new(
"000001.SZ".to_string(),
&[
market_row("2025-01-02", 10.0, 100),
market_row("2025-01-03", 11.0, 200),
current,
],
);
let decision_date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
assert_eq!(
series.decision_close_moving_average(decision_date, 2),
Some(11.5)
);
assert_eq!(
series.moving_average(decision_date, 2, PriceField::Close),
Some((11.0 + 9_999.0) / 2.0)
);
}
#[test]
fn current_close_average_uses_backward_adjustment_factor_and_current_base() {
let dates = [
NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-03", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap(),
];
let factors = [1.0, 1.0, 2.0];
let closes = [10.0, 11.0, 6.0];
let data = DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "000001.SZ".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: Some(dates[0]),
delisted_at: None,
status: "active".to_string(),
}],
dates
.iter()
.zip(closes)
.map(|(date, close)| market_row(&date.format("%Y-%m-%d").to_string(), close, 100))
.collect(),
dates
.iter()
.zip(factors)
.map(|(date, factor)| DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: BTreeMap::from([("adjustment_factor_backward1".into(), factor)]),
})
.collect(),
Vec::new(),
dates
.iter()
.map(|date| BenchmarkSnapshot {
date: *date,
benchmark: "000852.SH".to_string(),
open: 100.0,
close: 100.0,
prev_close: 100.0,
volume: 1_000_000,
})
.collect(),
)
.expect("dataset");
assert_eq!(
data.market_current_numeric_moving_average(dates[2], "000001.SZ", "close", 3),
Some(5.5)
);
assert_eq!(
data.market_decision_numeric_moving_average(dates[2], "000001.SZ", "close", 2),
Some(10.5)
);
assert_ne!(
data.market_current_numeric_moving_average(dates[2], "000001.SZ", "close", 3),
data.market_moving_average(dates[2], "000001.SZ", 3, PriceField::Close)
);
}
#[test]
fn adjusted_close_average_normalization_prevents_strict_crossover_drift() {
let pattern = [
2.953, 1.093, 2.717, 1.579, 1.289, 1.236, 1.617, 2.632, 1.361, 2.163,
];
let start = NaiveDate::parse_from_str("2025-01-01", "%Y-%m-%d").unwrap();
let values = (0..30)
.map(|index| pattern[index % pattern.len()])
.collect::<Vec<_>>();
let series = AdjustedCloseSeries {
dates: (0..30)
.map(|index| start + chrono::Duration::days(index as i64))
.collect(),
backward_factors: vec![Some(1.0); 30],
back_adjusted_closes: values.iter().copied().map(Some).collect(),
back_adjusted_close_prefix: prefix_sums(&values),
missing_back_adjusted_close_prefix: vec![0; 31],
};
let date = *series.dates.last().expect("last date");
assert_eq!(
series.current_moving_average(date, 10),
series.current_moving_average(date, 30)
);
}
#[test]
fn future_missing_adjustment_factor_does_not_invalidate_historical_window() {
let dates = [
NaiveDate::parse_from_str("2025-01-02", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-03", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap(),
NaiveDate::parse_from_str("2025-01-07", "%Y-%m-%d").unwrap(),
];
let data = DataSet::from_components(
vec![Instrument {
symbol: "000001.SZ".to_string(),
name: "000001.SZ".to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: Some(dates[0]),
delisted_at: None,
status: "active".to_string(),
}],
dates
.iter()
.enumerate()
.map(|(index, date)| {
market_row(
&date.format("%Y-%m-%d").to_string(),
10.0 + index as f64,
100,
)
})
.collect(),
dates
.iter()
.map(|date| DailyFactorSnapshot {
date: *date,
symbol: "000001.SZ".to_string(),
market_cap_bn: 10.0,
free_float_cap_bn: 8.0,
pe_ttm: 10.0,
turnover_ratio: None,
effective_turnover_ratio: None,
extra_factors: if *date == dates[3] {
BTreeMap::new()
} else {
BTreeMap::from([("adjustment_factor_backward1".into(), 1.0)])
},
})
.collect(),
Vec::new(),
dates
.iter()
.map(|date| BenchmarkSnapshot {
date: *date,
benchmark: "000852.SH".to_string(),
open: 100.0,
close: 100.0,
prev_close: 100.0,
volume: 1_000_000,
})
.collect(),
)
.expect("dataset");
assert_eq!(
data.market_current_numeric_moving_average(dates[2], "000001.SZ", "close", 3),
Some(11.0)
);
assert_eq!(
data.market_current_numeric_moving_average(dates[3], "000001.SZ", "close", 3),
None
);
}
#[test]
fn decision_volume_average_ignores_paused_zero_volume_days() {
let mut paused = market_row("2025-01-03", 11.0, 0);
paused.paused = true;
let series = SymbolPriceSeries::new(
"000001.SZ".to_string(),
&[
market_row("2025-01-02", 10.0, 100),
paused,
market_row("2025-01-06", 12.0, 300),
market_row("2025-01-07", 13.0, 10_000),
],
);
assert_eq!(
series.decision_volume_moving_average(
NaiveDate::parse_from_str("2025-01-07", "%Y-%m-%d").unwrap(),
2
),
Some(200.0)
);
assert_eq!(
series.decision_volume_moving_average(
NaiveDate::parse_from_str("2025-01-07", "%Y-%m-%d").unwrap(),
3
),
None
);
}
#[test]
fn eligible_universe_uses_decision_market_cap_same_date() {
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
let instrument = |symbol: &str| Instrument {
symbol: symbol.to_string(),
name: symbol.to_string(),
board: if symbol.ends_with(".SH") { "SH" } else { "SZ" }.to_string(),
round_lot: 100,
listed_at: Some(NaiveDate::parse_from_str("2020-01-01", "%Y-%m-%d").unwrap()),
delisted_at: None,
status: "active".to_string(),
};
let market = |symbol: &str, prev_close: f64, close: f64| DailyMarketSnapshot {
date,
symbol: symbol.to_string(),
timestamp: Some("2025-01-06 10:18:00".to_string()),
day_open: prev_close,
open: prev_close,
high: close.max(prev_close),
low: close.min(prev_close),
close,
last_price: prev_close,
bid1: prev_close,
ask1: prev_close,
prev_close,
volume: 100_000,
minute_volume: 1_000,
bid1_volume: 1_000,
ask1_volume: 1_000,
trading_phase: Some("continuous".to_string()),
paused: false,
upper_limit: prev_close * 1.1,
lower_limit: prev_close * 0.9,
price_tick: 0.01,
};
let factor =
|symbol: &str, market_cap_bn: f64, free_float_cap_bn: f64| DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
market_cap_bn,
free_float_cap_bn,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
extra_factors: BTreeMap::new(),
};
let candidate = |symbol: &str| CandidateEligibility {
date,
symbol: symbol.to_string(),
is_st: false,
is_star_st: false,
is_new_listing: false,
is_paused: false,
allow_buy: true,
allow_sell: true,
is_kcb: false,
is_one_yuan: false,
risk_level_code: None,
};
let data = DataSet::from_components(
vec![instrument("000001.SZ"), instrument("000002.SZ")],
vec![
market("000001.SZ", 10.0, 20.0),
market("000002.SZ", 10.0, 10.0),
],
vec![
factor("000001.SZ", 12.0, 4.0),
factor("000002.SZ", 10.0, 5.0),
],
vec![candidate("000001.SZ"), candidate("000002.SZ")],
vec![BenchmarkSnapshot {
date,
benchmark: "000852.SH".to_string(),
open: 100.0,
close: 101.0,
prev_close: 99.0,
volume: 1_000_000,
}],
)
.expect("dataset");
let rows = data.eligible_universe_on(date);
assert_eq!(rows.len(), 2);
assert_eq!(rows[0].symbol, "000002.SZ");
assert!((rows[0].market_cap_bn - 10.0).abs() < 1e-9);
assert_eq!(rows[1].symbol, "000001.SZ");
assert!((rows[1].market_cap_bn - 12.0).abs() < 1e-9);
assert!((rows[1].free_float_cap_bn - 4.0).abs() < 1e-9);
}
#[test]
fn eligible_universe_does_not_require_candidate_risk_state_when_selection_risk_is_disabled() {
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
let symbol = "000001.SZ";
let data = DataSet::from_components(
vec![Instrument {
symbol: symbol.to_string(),
name: symbol.to_string(),
board: "SZ".to_string(),
round_lot: 100,
listed_at: Some(NaiveDate::parse_from_str("2020-01-01", "%Y-%m-%d").unwrap()),
delisted_at: None,
status: "active".to_string(),
}],
vec![DailyMarketSnapshot {
date,
symbol: symbol.to_string(),
timestamp: Some("2025-01-06 10:18:00".to_string()),
day_open: 10.0,
open: 10.0,
high: 10.2,
low: 9.8,
close: 10.1,
last_price: 10.1,
bid1: 10.0,
ask1: 10.1,
prev_close: 10.0,
volume: 100_000,
minute_volume: 1_000,
bid1_volume: 1_000,
ask1_volume: 1_000,
trading_phase: Some("continuous".to_string()),
paused: false,
upper_limit: 11.0,
lower_limit: 9.0,
price_tick: 0.01,
}],
vec![DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
market_cap_bn: 10.0,
free_float_cap_bn: 9.0,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
extra_factors: BTreeMap::new(),
}],
Vec::new(),
vec![BenchmarkSnapshot {
date,
benchmark: "000852.SH".to_string(),
open: 100.0,
close: 101.0,
prev_close: 99.0,
volume: 1_000_000,
}],
)
.expect("dataset");
assert_eq!(
data.eligible_universe_on(date)
.iter()
.map(|row| row.symbol.as_str())
.collect::<Vec<_>>(),
vec![symbol]
);
assert_eq!(
data.eligible_universe_on_with_risk_config(date, &FidcRiskControlConfig::default())
.iter()
.map(|row| row.symbol.as_str())
.collect::<Vec<_>>(),
vec![symbol],
"execution-risk defaults must not make selection depend on candidate risk facts"
);
let mut selection_risk_config = FidcRiskControlConfig::default();
selection_risk_config.static_rules.reject_st_selection = true;
assert!(
data.eligible_universe_on_with_risk_config(date, &selection_risk_config)
.is_empty(),
"explicit selection risk must reject when required candidate facts are missing"
);
}
#[test]
fn eligible_universe_can_use_configured_risk_policy() {
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
let symbol = "688001.SH";
let data = DataSet::from_components(
vec![Instrument {
symbol: symbol.to_string(),
name: symbol.to_string(),
board: "SH".to_string(),
round_lot: 100,
listed_at: Some(NaiveDate::parse_from_str("2020-01-01", "%Y-%m-%d").unwrap()),
delisted_at: None,
status: "active".to_string(),
}],
vec![DailyMarketSnapshot {
date,
symbol: symbol.to_string(),
timestamp: Some("2025-01-06 10:18:00".to_string()),
day_open: 10.0,
open: 10.0,
high: 10.2,
low: 9.8,
close: 10.1,
last_price: 10.1,
bid1: 10.0,
ask1: 10.1,
prev_close: 10.0,
volume: 100_000,
minute_volume: 1_000,
bid1_volume: 1_000,
ask1_volume: 1_000,
trading_phase: Some("continuous".to_string()),
paused: false,
upper_limit: 11.0,
lower_limit: 9.0,
price_tick: 0.01,
}],
vec![DailyFactorSnapshot {
date,
symbol: symbol.to_string(),
market_cap_bn: 10.0,
free_float_cap_bn: 9.0,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
extra_factors: BTreeMap::new(),
}],
vec![CandidateEligibility {
date,
symbol: symbol.to_string(),
is_st: false,
is_star_st: false,
is_new_listing: false,
is_paused: false,
allow_buy: true,
allow_sell: true,
is_kcb: true,
is_one_yuan: false,
risk_level_code: None,
}],
vec![BenchmarkSnapshot {
date,
benchmark: "000852.SH".to_string(),
open: 100.0,
close: 101.0,
prev_close: 99.0,
volume: 1_000_000,
}],
)
.expect("dataset");
assert_eq!(data.eligible_universe_on(date).len(), 1);
let mut risk_config = FidcRiskControlConfig::default();
risk_config.static_rules.reject_kcb_selection = true;
assert!(
data.eligible_universe_on_with_risk_config(date, &risk_config)
.is_empty()
);
risk_config.static_rules.reject_kcb_selection = false;
let rows = data.eligible_universe_on_with_risk_config(date, &risk_config);
assert_eq!(rows.len(), 1);
assert_eq!(rows[0].symbol, symbol);
}
#[test]
fn decision_market_cap_uses_factor_date_snapshot_without_price_reconstruction() {
let date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
let factor = DailyFactorSnapshot {
date,
symbol: "000001.SZ".to_string(),
market_cap_bn: 12.0,
free_float_cap_bn: 4.0,
pe_ttm: 10.0,
turnover_ratio: Some(1.0),
effective_turnover_ratio: Some(1.0),
extra_factors: BTreeMap::new(),
};
assert!((decision_market_cap_bn(&factor) - 12.0).abs() < 1e-9);
assert!((decision_free_float_cap_bn(&factor) - 4.0).abs() < 1e-9);
}
#[test]
fn benchmark_decision_close_windows_exclude_current_close() {
let series = BenchmarkPriceSeries::new(&[
benchmark_row("2025-01-02", 100.0),
benchmark_row("2025-01-03", 200.0),
benchmark_row("2025-01-06", 9_999.0),
]);
let decision_date = NaiveDate::parse_from_str("2025-01-06", "%Y-%m-%d").unwrap();
assert_eq!(series.decision_close(decision_date), Some(9_998.0));
assert_eq!(
series.decision_moving_average(decision_date, 2),
Some(150.0)
);
assert_eq!(
series.decision_values_for(decision_date, 2, PriceField::Close),
vec![100.0, 200.0]
);
assert_eq!(
series.moving_average(decision_date, 2),
Some((200.0 + 9_999.0) / 2.0)
);
}
}
Reference in New Issue View Git Blame Copy Permalink