Example of using the XGBoost CV with custom function (Python)

[YojanaGadiya]

Hello XGBoost team,

I am trying to use the XGBoost CV module with Optuna for parameter optimization with the following code:

def kappa_scorer(y_true, y_pred):
    """Kappa scorer for the XGBoost model."""
    return "kappa", cohen_kappa_score(y_true, y_pred)


def objective_xgboost(trial, study_name, X_train, y_train, label_to_idx, exp_type: str):
    """Objective function for the XGBoost classifier with final eval metric as Kappa."""
    params = {
        "verbosity": 0,
        "objective": "multi:softmax",
        "num_class": len(label_to_idx),
        "feval": kappa_scorer,
        "n_estimators": 1000,
        "eta": trial.suggest_float("learning_rate", 1e-2, 0.1, log=True),
        "max_depth": trial.suggest_int("max_depth", 2, 10),
        "colsample_bytree": trial.suggest_float(
            "colsample_bytree", 0.1, 0.7
        ),  # Percentage of features used per tree.
        "disable_default_eval_metric": 1,
    }

    # Training data
    y_train = y_train.map(label_to_idx)

    dtrain = xgb.DMatrix(X_train, label=y_train)

    # Optimization of kappa scoe
    pruning_callback = optuna.integration.XGBoostPruningCallback(trial, "test-kappa")
    xgboost_model = xgb.cv(
        params, dtrain, callbacks=[pruning_callback], seed=SEED, nfold=5
    )

    # Save model
    trial_path = _generate_dirs(exp_type)
    save_xgboost_trial(
        trial=trial, model=xgboost_model, study_name=study_name, trial_dir=trial_path
    )

    mean_kappa = xgboost_model["test-kappa-mean"].values[-1]  # Optimized for kappa

    return mean_kappa

However, I am not able to extract the metric results from the trained model and stable upon an error: KeyError: 'test-kappa'. Is there any pointer to what I am doing wrong?

Thank you.