From 25a15d36a9ce409aaa3bf58d7b560f84d9d1d9e9 Mon Sep 17 00:00:00 2001 From: Anna Levenberg Date: Mon, 1 Apr 2024 17:02:07 -0400 Subject: [PATCH] feat(batch/simple): add a simple cloud batch example (#321) * feat(batch/simple): add a simple cloud batch example * try adding batch/simple to ci * add custom dockerfile and cloudbuild * remove docker files * revert unrelated formating changes * address comments * Update batch/simple/simple.cc Co-authored-by: Carlos O'Ryan * remove the json dependency and add error handling for resource exhausted * add a comment about printing * format fixg --------- Co-authored-by: Carlos O'Ryan --- CMakeLists.txt | 1 + batch/simple/CMakeLists.txt | 27 +++++++++ batch/simple/README.md | 42 ++++++++++++++ batch/simple/hello-world-container.json | 42 ++++++++++++++ batch/simple/simple.cc | 73 +++++++++++++++++++++++++ batch/simple/vcpkg.json | 13 +++++ 6 files changed, 198 insertions(+) create mode 100644 batch/simple/CMakeLists.txt create mode 100644 batch/simple/README.md create mode 100644 batch/simple/hello-world-container.json create mode 100644 batch/simple/simple.cc create mode 100644 batch/simple/vcpkg.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 7878e014..16a6a310 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ include(ExternalProject) set(samples # cmake-format: sort + batch/simple bigquery/write cloud-run-hello-world gcs-fast-transfers diff --git a/batch/simple/CMakeLists.txt b/batch/simple/CMakeLists.txt new file mode 100644 index 00000000..424d260e --- /dev/null +++ b/batch/simple/CMakeLists.txt @@ -0,0 +1,27 @@ +# ~~~ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +cmake_minimum_required(VERSION 3.20) + +# Define the project name and where to report bugs. +set(PACKAGE_BUGREPORT + "https://github.com/GoogleCloudPlatform/cpp-samples/issues") +project(cpp-samples-batch CXX) + +find_package(google_cloud_cpp_batch REQUIRED) + +add_executable(simple simple.cc) +target_link_libraries("simple" PRIVATE google-cloud-cpp::batch) diff --git a/batch/simple/README.md b/batch/simple/README.md new file mode 100644 index 00000000..834d35c0 --- /dev/null +++ b/batch/simple/README.md @@ -0,0 +1,42 @@ +# Using Cloud Batch + +This example shows how to take a job.json and run a Cloud Batch job using C++. + +If you are not familiar with the Batch API, we recommend you first read the +[API overview] before starting this guide. + +## Compiling the Example + +This project uses `vcpkg` to install its dependencies. Clone `vcpkg` in your +`$HOME`: + +```shell +git clone -C $HOME https://github.com/microsoft/vcpkg.git +``` + +Install the typical development tools, on Ubuntu you would use: + +```shell +apt update && apt install -y build-essential cmake git ninja-build pkg-config g++ curl tar zip unzip +``` + +In this directory compile the dependencies and the code, this can take as long +as an hour, depending on the performance of your workstation: + +```shell +cd cpp-samples/batch/simple +cmake -S . -B .build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$HOME/vcpkg/scripts/buildsystems/vcpkg.cmake +cmake --build .build +``` + +## Run the sample + +Run the example, replace the `[PROJECT ID]` placeholder with the id of your +project: + +```shell +.build/simple [PROJECT ID] us-central1 test-container-run hello-world-container.json +``` + +[api overview]: https://cloud.google.com/batch/docs diff --git a/batch/simple/hello-world-container.json b/batch/simple/hello-world-container.json new file mode 100644 index 00000000..ef938666 --- /dev/null +++ b/batch/simple/hello-world-container.json @@ -0,0 +1,42 @@ +{ + "taskGroups": [ + { + "taskSpec": { + "runnables": [ + { + "container": { + "imageUri": "gcr.io/google-containers/busybox", + "entrypoint": "/bin/sh", + "commands": [ + "-c", + "echo Hello world! This is task ${BATCH_TASK_INDEX}. This job has a total of ${BATCH_TASK_COUNT} tasks." + ] + } + } + ], + "computeResource": { + "cpuMilli": 2000, + "memoryMib": 16 + }, + "maxRetryCount": 2, + "maxRunDuration": "3600s" + }, + "taskCount": 4, + "parallelism": 2 + } + ], + "allocationPolicy": { + "instances": [ + { + "policy": { "machineType": "e2-standard-4" } + } + ] + }, + "labels": { + "department": "finance", + "env": "testing" + }, + "logsPolicy": { + "destination": "CLOUD_LOGGING" + } +} diff --git a/batch/simple/simple.cc b/batch/simple/simple.cc new file mode 100644 index 00000000..959c24f0 --- /dev/null +++ b/batch/simple/simple.cc @@ -0,0 +1,73 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +int main(int argc, char* argv[]) try { + if (argc != 5) { + std::cerr << "Usage: " << argv[0] + << " \n"; + return 1; + } + + namespace batch = ::google::cloud::batch_v1; + + std::string const project_id = argv[1]; + auto const location = google::cloud::Location(argv[1], argv[2]); + std::string const job_id = argv[3]; + std::string const job_file = argv[4]; + + // Parse the json and convert into protobuf format. + std::ifstream file(job_file, std::ios::in); + if (!file.is_open()) { + std::cout << "Failed to open JSON file: " << job_file << '\n'; + return 0; + } + auto contents = std::string{std::istreambuf_iterator(file), {}}; + google::cloud::batch::v1::Job job; + google::protobuf::util::JsonParseOptions options; + google::protobuf::util::Status status = + google::protobuf::util::JsonStringToMessage(contents, &job, options); + if (!status.ok()) throw status; + + // Create the cloud batch client. + auto client = batch::BatchServiceClient(batch::MakeBatchServiceConnection()); + + // Create a job. + auto response = client.CreateJob(location.FullName(), job, job_id); + + if (!response) { + if (response.status().code() == + google::cloud::StatusCode::kResourceExhausted) { + std::cout << "There already exists a job for the parent `" + << location.FullName() << "` and job_id: `" << job_id + << "`. Please try again with a new job id.\n"; + return 0; + } + throw std::move(response).status(); + } + + // On success, print the job. + std::cout << "Job : " << response->DebugString() << "\n"; + return 0; +} catch (google::cloud::Status const& status) { + std::cerr << "google::cloud::Status thrown: " << status << "\n"; + return 1; +} catch (google::protobuf::util::Status const& status) { + std::cerr << "google::protobuf::util::Status thrown: " << status << "\n"; + return 1; +} diff --git a/batch/simple/vcpkg.json b/batch/simple/vcpkg.json new file mode 100644 index 00000000..7f84f41a --- /dev/null +++ b/batch/simple/vcpkg.json @@ -0,0 +1,13 @@ +{ + "name": "gcp-cpp-samples-batch", + "version-string": "unversioned", + "homepage": "https://github.com/GoogleCloudPlatform/cpp-samples/", + "description": "An example using the Cloud Batch API", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": ["batch"] + } + ] + }