diff --git a/tests/README.md b/tests/README.md index 7d54b78cb86e..a118e791813f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,9 +10,7 @@ facilities. dependencies for tests, see conda files in `ci_build`. * python-gpu: Similar to python tests, but for GPU. * travis: CI facilities for Travis. - * distributed: Test for distributed system. - * benchmark: Legacy benchmark code. There are a number of benchmark projects for - XGBoost with much better configurations. + * test_distributed: Test for distributed systems including spark and dask. # Others * pytest.ini: Describes the `pytest` marker for python tests, some markers are generated diff --git a/tests/benchmark/benchmark_linear.py b/tests/benchmark/benchmark_linear.py deleted file mode 100644 index cb51417140f7..000000000000 --- a/tests/benchmark/benchmark_linear.py +++ /dev/null @@ -1,69 +0,0 @@ -#pylint: skip-file -import argparse -import xgboost as xgb -import numpy as np -from sklearn.datasets import make_classification -from sklearn.model_selection import train_test_split -import time -import ast - -rng = np.random.RandomState(1994) - - -def run_benchmark(args): - - try: - dtest = xgb.DMatrix('dtest.dm') - dtrain = xgb.DMatrix('dtrain.dm') - - if not (dtest.num_col() == args.columns \ - and dtrain.num_col() == args.columns): - raise ValueError("Wrong cols") - if not (dtest.num_row() == args.rows * args.test_size \ - and dtrain.num_row() == args.rows * (1-args.test_size)): - raise ValueError("Wrong rows") - except: - - print("Generating dataset: {} rows * {} columns".format(args.rows, args.columns)) - print("{}/{} test/train split".format(args.test_size, 1.0 - args.test_size)) - tmp = time.time() - X, y = make_classification(args.rows, n_features=args.columns, n_redundant=0, n_informative=args.columns, n_repeated=0, random_state=7) - if args.sparsity < 1.0: - X = np.array([[np.nan if rng.uniform(0, 1) < args.sparsity else x for x in x_row] for x_row in X]) - - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=args.test_size, random_state=7) - print ("Generate Time: %s seconds" % (str(time.time() - tmp))) - tmp = time.time() - print ("DMatrix Start") - dtrain = xgb.DMatrix(X_train, y_train) - dtest = xgb.DMatrix(X_test, y_test, nthread=-1) - print ("DMatrix Time: %s seconds" % (str(time.time() - tmp))) - - dtest.save_binary('dtest.dm') - dtrain.save_binary('dtrain.dm') - - param = {'objective': 'binary:logistic','booster':'gblinear'} - if args.params != '': - param.update(ast.literal_eval(args.params)) - - param['updater'] = args.updater - print("Training with '%s'" % param['updater']) - tmp = time.time() - xgb.train(param, dtrain, args.iterations, evals=[(dtrain,"train")], early_stopping_rounds = args.columns) - print ("Train Time: %s seconds" % (str(time.time() - tmp))) - -parser = argparse.ArgumentParser() -parser.add_argument('--updater', default='coord_descent') -parser.add_argument('--sparsity', type=float, default=0.0) -parser.add_argument('--lambda', type=float, default=1.0) -parser.add_argument('--tol', type=float, default=1e-5) -parser.add_argument('--alpha', type=float, default=1.0) -parser.add_argument('--rows', type=int, default=1000000) -parser.add_argument('--iterations', type=int, default=10000) -parser.add_argument('--columns', type=int, default=50) -parser.add_argument('--test_size', type=float, default=0.25) -parser.add_argument('--standardise', type=bool, default=False) -parser.add_argument('--params', default='', help='Provide additional parameters as a Python dict string, e.g. --params \"{\'max_depth\':2}\"') -args = parser.parse_args() - -run_benchmark(args) diff --git a/tests/benchmark/benchmark_tree.py b/tests/benchmark/benchmark_tree.py deleted file mode 100644 index 380e03463eab..000000000000 --- a/tests/benchmark/benchmark_tree.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Run benchmark on the tree booster.""" - -import argparse -import ast -import time - -import numpy as np -import xgboost as xgb - -RNG = np.random.RandomState(1994) - - -def run_benchmark(args): - """Runs the benchmark.""" - try: - dtest = xgb.DMatrix('dtest.dm') - dtrain = xgb.DMatrix('dtrain.dm') - - if not (dtest.num_col() == args.columns - and dtrain.num_col() == args.columns): - raise ValueError("Wrong cols") - if not (dtest.num_row() == args.rows * args.test_size - and dtrain.num_row() == args.rows * (1 - args.test_size)): - raise ValueError("Wrong rows") - except: - print("Generating dataset: {} rows * {} columns".format(args.rows, args.columns)) - print("{}/{} test/train split".format(args.test_size, 1.0 - args.test_size)) - tmp = time.time() - X = RNG.rand(args.rows, args.columns) - y = RNG.randint(0, 2, args.rows) - if 0.0 < args.sparsity < 1.0: - X = np.array([[np.nan if RNG.uniform(0, 1) < args.sparsity else x for x in x_row] - for x_row in X]) - - train_rows = int(args.rows * (1.0 - args.test_size)) - test_rows = int(args.rows * args.test_size) - X_train = X[:train_rows, :] - X_test = X[-test_rows:, :] - y_train = y[:train_rows] - y_test = y[-test_rows:] - print("Generate Time: %s seconds" % (str(time.time() - tmp))) - del X, y - - tmp = time.time() - print("DMatrix Start") - dtrain = xgb.DMatrix(X_train, y_train, nthread=-1) - dtest = xgb.DMatrix(X_test, y_test, nthread=-1) - print("DMatrix Time: %s seconds" % (str(time.time() - tmp))) - del X_train, y_train, X_test, y_test - - dtest.save_binary('dtest.dm') - dtrain.save_binary('dtrain.dm') - - param = {'objective': 'binary:logistic'} - if args.params != '': - param.update(ast.literal_eval(args.params)) - - param['tree_method'] = args.tree_method - print("Training with '%s'" % param['tree_method']) - tmp = time.time() - xgb.train(param, dtrain, args.iterations, evals=[(dtest, "test")]) - print("Train Time: %s seconds" % (str(time.time() - tmp))) - - -def main(): - """The main function. - - Defines and parses command line arguments and calls the benchmark. - """ - parser = argparse.ArgumentParser() - parser.add_argument('--tree_method', default='gpu_hist') - parser.add_argument('--sparsity', type=float, default=0.0) - parser.add_argument('--rows', type=int, default=1000000) - parser.add_argument('--columns', type=int, default=50) - parser.add_argument('--iterations', type=int, default=500) - parser.add_argument('--test_size', type=float, default=0.25) - parser.add_argument('--params', default='', - help='Provide additional parameters as a Python dict string, e.g. --params ' - '\"{\'max_depth\':2}\"') - args = parser.parse_args() - - run_benchmark(args) - - -if __name__ == '__main__': - main() diff --git a/tests/benchmark/generate_libsvm.py b/tests/benchmark/generate_libsvm.py deleted file mode 100644 index be152df39af4..000000000000 --- a/tests/benchmark/generate_libsvm.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Generate synthetic data in LIBSVM format.""" - -import argparse -import io -import time - -import numpy as np -from sklearn.datasets import make_classification -from sklearn.model_selection import train_test_split - -RNG = np.random.RandomState(2019) - - -def generate_data(args): - """Generates the data.""" - print("Generating dataset: {} rows * {} columns".format(args.rows, args.columns)) - print("Sparsity {}".format(args.sparsity)) - print("{}/{} train/test split".format(1.0 - args.test_size, args.test_size)) - - tmp = time.time() - n_informative = args.columns * 7 // 10 - n_redundant = args.columns // 10 - n_repeated = args.columns // 10 - print("n_informative: {}, n_redundant: {}, n_repeated: {}".format(n_informative, n_redundant, - n_repeated)) - x, y = make_classification(n_samples=args.rows, n_features=args.columns, - n_informative=n_informative, n_redundant=n_redundant, - n_repeated=n_repeated, shuffle=False, random_state=RNG) - print("Generate Time: {} seconds".format(time.time() - tmp)) - - tmp = time.time() - x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=args.test_size, - random_state=RNG, shuffle=False) - print("Train/Test Split Time: {} seconds".format(time.time() - tmp)) - - tmp = time.time() - write_file('train.libsvm', x_train, y_train, args.sparsity) - print("Write Train Time: {} seconds".format(time.time() - tmp)) - - tmp = time.time() - write_file('test.libsvm', x_test, y_test, args.sparsity) - print("Write Test Time: {} seconds".format(time.time() - tmp)) - - -def write_file(filename, x_data, y_data, sparsity): - with open(filename, 'w') as f: - for x, y in zip(x_data, y_data): - write_line(f, x, y, sparsity) - - -def write_line(f, x, y, sparsity): - with io.StringIO() as line: - line.write(str(y)) - for i, col in enumerate(x): - if 0.0 < sparsity < 1.0: - if RNG.uniform(0, 1) > sparsity: - write_feature(line, i, col) - else: - write_feature(line, i, col) - line.write('\n') - f.write(line.getvalue()) - - -def write_feature(line, index, feature): - line.write(' ') - line.write(str(index)) - line.write(':') - line.write(str(feature)) - - -def main(): - """The main function. - - Defines and parses command line arguments and calls the generator. - """ - parser = argparse.ArgumentParser() - parser.add_argument('--rows', type=int, default=1000000) - parser.add_argument('--columns', type=int, default=50) - parser.add_argument('--sparsity', type=float, default=0.0) - parser.add_argument('--test_size', type=float, default=0.01) - args = parser.parse_args() - - generate_data(args) - - -if __name__ == '__main__': - main()