特徴量エンジニアリング

# ml-prediction チュートリアルの fetch() をそのまま使う
ky = fetch("dky", start_date, end_date)
ra = fetch("dra", start_date, end_date)
se = fetch("dse", start_date, end_date)

df = (
    ky
    .merge(
        ra[['racekey', 'distance', 'track_type', 'turn_cd',
            'course_cd', 'racecourse_cd', 'turf_condition_cd']],
        on='racekey', how='left'
    )
    .merge(
        se[['racekey', 'horse_number', 'order_of_finish', 'idm',
            'horse_weight', 'confirmed_win_odds', 'confirmed_win_popularity']],
        on=['racekey', 'horse_number'], how='inner'
    )
)

# 数値として扱う列は明示的に変換しておく
for col in ['distance', 'bracket_number', 'horse_number', 'order_of_finish',
            'idm', 'horse_weight', 'confirmed_win_odds', 'confirmed_win_popularity']:
    df[col] = pd.to_numeric(df[col], errors='coerce')

import pandas as pd
import numpy as np

def create_past_performance_features(df):
    """過去成績から特徴量を作成"""

    # 馬ごとにソート（日付順）
    df = df.sort_values(['horse_id', 'racedate'])

    # 過去N走の成績を集約
    for n in [3, 5, 10]:
        # 過去N走の平均着順
        df[f'avg_finish_{n}'] = df.groupby('horse_id')['order_of_finish'].transform(
            lambda x: x.shift(1).rolling(n, min_periods=1).mean()
        )

        # 過去N走の勝率
        df[f'win_rate_{n}'] = df.groupby('horse_id')['order_of_finish'].transform(
            lambda x: (x.shift(1) == 1).rolling(n, min_periods=1).mean()
        )

        # 過去N走の複勝率（3着以内）
        df[f'place_rate_{n}'] = df.groupby('horse_id')['order_of_finish'].transform(
            lambda x: (x.shift(1) <= 3).rolling(n, min_periods=1).mean()
        )

    return df

df = create_past_performance_features(df)
print(f"作成した特徴量: {[c for c in df.columns if 'avg_' in c or 'rate_' in c]}")

def create_relative_features(df):
    """レース内での相対的な位置を特徴量化"""

    # IDMの偏差値化（レース内）
    df['idm_zscore'] = df.groupby('racekey')['idm'].transform(
        lambda x: (x - x.mean()) / x.std()
    )

    # レース内での人気順位
    df['popularity_rank'] = df.groupby('racekey')['confirmed_win_popularity'].rank()

    # オッズの対数変換（スケール調整）
    df['log_odds'] = np.log1p(df['confirmed_win_odds'])

    # レース内でのオッズ偏差値
    df['odds_zscore'] = df.groupby('racekey')['log_odds'].transform(
        lambda x: (x - x.mean()) / x.std()
    )

    # 馬体重のレース内偏差値
    df['weight_zscore'] = df.groupby('racekey')['horse_weight'].transform(
        lambda x: (x - x.mean()) / x.std()
    )

    return df

df = create_relative_features(df)

def create_lag_features(df):
    """前走情報をラグ特徴量として作成"""

    df = df.sort_values(['horse_id', 'racedate'])

    # 前走からの間隔日数
    df['days_since_last'] = df.groupby('horse_id')['racedate'].transform(
        lambda x: pd.to_datetime(x).diff().dt.days
    )

    # 前走着順
    df['last_finish'] = df.groupby('horse_id')['order_of_finish'].shift(1)

    # 前走人気
    df['last_popularity'] = df.groupby('horse_id')['confirmed_win_popularity'].shift(1)

    # 前走との人気差（期待値の変化）
    df['popularity_change'] = df['confirmed_win_popularity'] - df['last_popularity']

    # 前走IDM
    df['last_idm'] = df.groupby('horse_id')['idm'].shift(1)

    # IDMの変化
    df['idm_change'] = df['idm'] - df['last_idm']

    # 前走馬体重
    df['last_weight'] = df.groupby('horse_id')['horse_weight'].shift(1)

    # 馬体重の変化
    df['weight_change'] = df['horse_weight'] - df['last_weight']

    return df

df = create_lag_features(df)

def create_course_aptitude_features(df):
    """コース適性の特徴量を作成"""

    # 距離カテゴリの作成
    df['distance_cat'] = pd.cut(
        df['distance'],
        bins=[0, 1400, 1800, 2200, 9999],
        labels=['sprint', 'mile', 'middle', 'long']
    )

    # 芝/ダート別成績（過去実績から計算）
    for track in ['turf', 'dirt']:
        track_mask = df['track_type'] == ('1' if track == 'turf' else '2')

        # 該当トラックでの勝率
        df[f'{track}_win_rate'] = df.groupby('horse_id').apply(
            lambda x: (x.loc[track_mask, 'order_of_finish'].shift(1) == 1).expanding().mean()
        ).reset_index(level=0, drop=True)

    # 距離別成績
    df['distance_win_rate'] = df.groupby(['horse_id', 'distance_cat'])['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 競馬場別成績
    df['course_win_rate'] = df.groupby(['horse_id', 'course_cd'])['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 今回距離と過去ベスト距離の差
    df['best_distance'] = df.groupby('horse_id').apply(
        lambda x: x.loc[x['order_of_finish'] == x['order_of_finish'].min(), 'distance'].iloc[0]
        if len(x) > 0 else np.nan
    ).reset_index(level=0, drop=True)
    df['distance_from_best'] = abs(df['distance'] - df['best_distance'])

    return df

df = create_course_aptitude_features(df)

def create_track_condition_features(df):
    """馬場状態への適性特徴量を作成"""

    # 馬場状態の数値化（良=1, 稍重=2, 重=3, 不良=4）
    condition_map = {'1': 1, '2': 2, '3': 3, '4': 4}
    df['condition_num'] = df['turf_condition_cd'].map(condition_map)

    # 重馬場適性（重・不良での成績）
    heavy_mask = df['condition_num'] >= 3
    df['heavy_track_rate'] = df.groupby('horse_id').apply(
        lambda x: (x.loc[heavy_mask, 'order_of_finish'].shift(1) <= 3).expanding().mean()
    ).reset_index(level=0, drop=True)

    # 良馬場での成績
    good_mask = df['condition_num'] == 1
    df['good_track_rate'] = df.groupby('horse_id').apply(
        lambda x: (x.loc[good_mask, 'order_of_finish'].shift(1) <= 3).expanding().mean()
    ).reset_index(level=0, drop=True)

    # 今回の馬場と得意馬場の一致度
    df['track_condition_match'] = np.where(
        (df['condition_num'] >= 3) & (df['heavy_track_rate'] > df['good_track_rate']),
        1,
        np.where(
            (df['condition_num'] == 1) & (df['good_track_rate'] > df['heavy_track_rate']),
            1,
            0
        )
    )

    return df

df = create_track_condition_features(df)

def create_jockey_trainer_features(df):
    """騎手・調教師関連の特徴量を作成"""

    # 騎手の直近成績（過去30日）
    df['jockey_recent_win_rate'] = df.groupby('jockey_cd')['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).rolling(20, min_periods=5).mean()
    )

    # 調教師の直近成績
    df['trainer_recent_win_rate'] = df.groupby('trainer_cd')['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).rolling(20, min_periods=5).mean()
    )

    # 騎手乗り替わりフラグ
    df['jockey_change'] = (
        df.groupby('horse_id')['jockey_cd'].shift(1) != df['jockey_cd']
    ).astype(int)

    # 騎手×馬の相性（過去の組み合わせ成績）
    df['jockey_horse_wins'] = df.groupby(['jockey_cd', 'horse_id'])['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().sum()
    )

    # 騎手×距離の相性
    df['jockey_distance_rate'] = df.groupby(['jockey_cd', 'distance_cat'])['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 調教師×競馬場の相性
    df['trainer_course_rate'] = df.groupby(['trainer_cd', 'course_cd'])['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    return df

df = create_jockey_trainer_features(df)

def create_trend_features(df):
    """成績のトレンドを特徴量化"""

    df = df.sort_values(['horse_id', 'racedate'])

    # 着順の移動平均（トレンド）
    df['finish_ma_3'] = df.groupby('horse_id')['order_of_finish'].transform(
        lambda x: x.shift(1).rolling(3, min_periods=1).mean()
    )
    df['finish_ma_5'] = df.groupby('horse_id')['order_of_finish'].transform(
        lambda x: x.shift(1).rolling(5, min_periods=1).mean()
    )

    # 短期 vs 長期のトレンド（上昇傾向かどうか）
    df['finish_trend'] = df['finish_ma_3'] - df['finish_ma_5']

    # IDMのトレンド
    df['idm_ma_3'] = df.groupby('horse_id')['idm'].transform(
        lambda x: x.shift(1).rolling(3, min_periods=1).mean()
    )
    df['idm_trend'] = df['idm'] - df['idm_ma_3']

    # 連続好走フラグ（過去3走連続3着以内）
    df['consecutive_places'] = df.groupby('horse_id')['order_of_finish'].transform(
        lambda x: (x.shift(1) <= 3).rolling(3, min_periods=3).sum() == 3
    ).astype(int)

    # 連続凡走フラグ（過去3走連続5着以下）
    df['consecutive_poor'] = df.groupby('horse_id')['order_of_finish'].transform(
        lambda x: (x.shift(1) > 5).rolling(3, min_periods=3).sum() == 3
    ).astype(int)

    return df

def create_rest_features(df):
    """休養・間隔に関する特徴量"""

    # 休養明けフラグ（90日以上の間隔）
    df['is_rest_return'] = (df['days_since_last'] >= 90).astype(int)

    # 連闘フラグ（14日以内）
    df['is_consecutive'] = (df['days_since_last'] <= 14).astype(int)

    # 中2週〜中4週（ベストな間隔）
    df['is_optimal_interval'] = (
        (df['days_since_last'] >= 14) & (df['days_since_last'] <= 35)
    ).astype(int)

    # 間隔のカテゴリ化
    df['interval_cat'] = pd.cut(
        df['days_since_last'],
        bins=[0, 14, 35, 90, 180, 9999],
        labels=['consecutive', 'short', 'optimal', 'rest', 'long_rest']
    )

    return df

df = create_trend_features(df)
df = create_rest_features(df)

def create_outer_stable_features(df):
    """外厩関連の相互作用特徴量を作成"""

    # 外厩×調教師別成績
    df['outer_trainer_winrate'] = df.groupby(
        ['outer_stable', 'trainer_cd']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 外厩×調教師別複勝率
    df['outer_trainer_place_rate'] = df.groupby(
        ['outer_stable', 'trainer_cd']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) <= 3).expanding().mean()
    )

    # 外厩×距離カテゴリ別成績
    df['outer_distance_winrate'] = df.groupby(
        ['outer_stable', 'distance_cat']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 外厩×距離カテゴリ別複勝率
    df['outer_distance_place_rate'] = df.groupby(
        ['outer_stable', 'distance_cat']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) <= 3).expanding().mean()
    )

    # 外厩×調教師×距離の3要素組み合わせ
    df['outer_trainer_dist_winrate'] = df.groupby(
        ['outer_stable', 'trainer_cd', 'distance_cat']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 外厩×競馬場別成績
    df['outer_course_winrate'] = df.groupby(
        ['outer_stable', 'course_cd']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) == 1).expanding().mean()
    )

    # 外厩からの休養明け成績
    df['outer_rest_return_rate'] = df.groupby('outer_stable').apply(
        lambda x: x.loc[x['is_rest_return'] == 1, 'order_of_finish'].shift(1).le(3).expanding().mean()
    ).reset_index(level=0, drop=True)

    return df

df = create_outer_stable_features(df)
print("外厩関連の特徴量を作成しました")

def create_pace_features(df):
    """展開予測に関する特徴量"""

    # レース内の脚質分布
    df['escape_count'] = df.groupby('racekey')['foottype_cd'].transform(
        lambda x: (x == 1).sum()  # 逃げ馬の数
    )
    df['front_count'] = df.groupby('racekey')['foottype_cd'].transform(
        lambda x: (x <= 2).sum()  # 先行馬以上の数
    )

    # 逃げ馬が1頭のみ（楽逃げ可能性）
    df['easy_escape'] = ((df['foottype_cd'] == 1) & (df['escape_count'] == 1)).astype(int)

    # 前残り有利度（先行馬が少ない = 前残りしやすい）
    df['front_advantage'] = 1 / (df['front_count'] + 1)

    # 差し有利度（先行馬が多い = 差しが決まりやすい）
    df['closer_advantage'] = np.where(
        df['foottype_cd'] >= 3,
        df['front_count'] / df.groupby('racekey')['horse_number'].transform('count'),
        0
    )

    # 自分の脚質と展開の相性
    df['pace_match'] = np.where(
        (df['foottype_cd'] <= 2) & (df['front_count'] <= 4),
        1,  # 先行馬で前が少ない
        np.where(
            (df['foottype_cd'] >= 3) & (df['front_count'] >= 5),
            1,  # 差し馬で前が多い
            0
        )
    )

    return df

df = create_pace_features(df)

def create_post_position_features(df):
    """枠順・馬番に関する特徴量"""

    # 頭数に対する馬番の相対位置
    df['relative_post'] = df['horse_number'] / df.groupby('racekey')['horse_number'].transform('max')

    # 内枠・外枠フラグ
    df['is_inner'] = (df['bracket_number'] <= 3).astype(int)
    df['is_outer'] = (df['bracket_number'] >= 7).astype(int)

    # コース×枠順の相性（右回り外枠不利など）
    # 回り（turn_cd: 1=右, 2=左, 9=直線）は /tra・/dra から取得する
    df['is_right_course'] = (df['turn_cd'].astype(str) == '1').astype(int)

    # 右回り外枠不利
    df['right_outer_disadvantage'] = (
        (df['is_right_course'] == 1) & (df['is_outer'] == 1)
    ).astype(int)

    # 距離×枠順（短距離は内枠有利）
    df['sprint_inner_advantage'] = (
        (df['distance'] <= 1400) & (df['is_inner'] == 1)
    ).astype(int)

    # 過去の枠順別成績
    df['post_position_rate'] = df.groupby(
        ['horse_id', 'bracket_number']
    )['order_of_finish'].transform(
        lambda x: (x.shift(1) <= 3).expanding().mean()
    )

    return df

df = create_post_position_features(df)

def handle_missing_values(df, feature_columns):
    """JRDBデータ特有の欠損値処理"""

    # 数値特徴量の欠損値処理
    numeric_cols = df[feature_columns].select_dtypes(include=[np.number]).columns

    # 成績系の欠損は-1で埋める（データなしを明示）
    rate_cols = [c for c in numeric_cols if 'rate' in c or 'winrate' in c]
    df[rate_cols] = df[rate_cols].fillna(-1)

    # 平均・偏差値系は0で埋める
    zscore_cols = [c for c in numeric_cols if 'zscore' in c or 'avg' in c]
    df[zscore_cols] = df[zscore_cols].fillna(0)

    # その他は中央値で埋める
    other_cols = [c for c in numeric_cols if c not in rate_cols + zscore_cols]
    df[other_cols] = df[other_cols].fillna(df[other_cols].median())

    # カテゴリ特徴量
    cat_cols = df[feature_columns].select_dtypes(include=['object', 'category']).columns
    df[cat_cols] = df[cat_cols].fillna('unknown')

    return df

df = handle_missing_values(df, feature_columns)

from sklearn.model_selection import KFold

def target_encode(df, cat_col, target_col, n_folds=5):
    """Target Encodingでカテゴリをエンコード"""

    # 全体の平均（スムージング用）
    global_mean = df[target_col].mean()

    # CV用のエンコード値を格納
    encoded = pd.Series(index=df.index, dtype=float)

    kf = KFold(n_splits=n_folds, shuffle=True, random_state=42)

    for train_idx, val_idx in kf.split(df):
        # 訓練データでの平均を計算
        train_means = df.iloc[train_idx].groupby(cat_col)[target_col].mean()

        # バリデーションデータに適用
        encoded.iloc[val_idx] = df.iloc[val_idx][cat_col].map(train_means)

    # 欠損値は全体平均で埋める
    encoded = encoded.fillna(global_mean)

    return encoded

# 騎手・調教師・外厩などをTarget Encoding
df['jockey_target_enc'] = target_encode(df, 'jockey_cd', 'is_win')
df['trainer_target_enc'] = target_encode(df, 'trainer_cd', 'is_win')
df['outer_stable_target_enc'] = target_encode(df, 'outer_stable', 'is_win')

from sklearn.feature_selection import mutual_info_classif
import lightgbm as lgb

def select_features(X, y, n_features=50):
    """特徴量選択を実行"""

    # 1. 相互情報量による選択
    mi_scores = mutual_info_classif(X.fillna(0), y, random_state=42)
    mi_importance = pd.DataFrame({
        'feature': X.columns,
        'mi_score': mi_scores
    }).sort_values('mi_score', ascending=False)

    # 2. LightGBMの特徴量重要度
    model = lgb.LGBMClassifier(n_estimators=100, random_state=42, verbose=-1)
    model.fit(X.fillna(0), y)

    lgb_importance = pd.DataFrame({
        'feature': X.columns,
        'lgb_importance': model.feature_importances_
    }).sort_values('lgb_importance', ascending=False)

    # 3. 両方で上位の特徴量を選択
    top_mi = set(mi_importance.head(n_features * 2)['feature'])
    top_lgb = set(lgb_importance.head(n_features * 2)['feature'])

    # 共通する特徴量を優先
    common_features = top_mi & top_lgb

    # 足りない分は重要度順に追加
    remaining = n_features - len(common_features)
    if remaining > 0:
        additional = lgb_importance[
            ~lgb_importance['feature'].isin(common_features)
        ].head(remaining)['feature'].tolist()
        selected = list(common_features) + additional
    else:
        selected = list(common_features)[:n_features]

    print(f"選択された特徴量数: {len(selected)}")
    return selected

selected_features = select_features(X, y, n_features=50)
X_selected = X[selected_features]

def create_all_features(df):
    """全ての特徴量を作成するパイプライン"""

    print("Step 1: 基本特徴量の作成...")
    df = create_past_performance_features(df)
    df = create_relative_features(df)
    df = create_lag_features(df)

    print("Step 2: ドメイン知識特徴量の作成...")
    df = create_course_aptitude_features(df)
    df = create_track_condition_features(df)
    df = create_jockey_trainer_features(df)

    print("Step 3: 時系列特徴量の作成...")
    df = create_trend_features(df)
    df = create_rest_features(df)

    print("Step 4: 相互作用特徴量の作成...")
    df = create_outer_stable_features(df)
    df = create_pace_features(df)
    df = create_post_position_features(df)

    print("Step 5: 欠損値処理...")
    feature_columns = [c for c in df.columns if c not in ['racekey', 'horse_id', 'order_of_finish']]
    df = handle_missing_values(df, feature_columns)

    print(f"完了! 作成された特徴量数: {len(feature_columns)}")

    return df

# 実行
df = create_all_features(df)