From 92cc70fb1eca6ca19394540ea9cfe9757a10e8f5 Mon Sep 17 00:00:00 2001 From: Anna Levenberg Date: Mon, 13 May 2024 20:32:42 -0400 Subject: [PATCH] impl(batch/cpp_application): add a cloud batch example for a cpp application --- .gitignore | 1 + CMakeLists.txt | 1 + batch/cpp_application/CMakeLists.txt | 27 ++++ batch/cpp_application/README.md | 124 +++++++++++++++ batch/cpp_application/application.json | 36 +++++ .../application/CMakeLists.txt | 24 +++ batch/cpp_application/application/Dockerfile | 51 ++++++ batch/cpp_application/application/README.md | 21 +++ batch/cpp_application/application/src/main.cc | 17 ++ batch/cpp_application/main.cc | 145 ++++++++++++++++++ batch/cpp_application/vcpkg.json | 13 ++ 11 files changed, 460 insertions(+) create mode 100644 batch/cpp_application/CMakeLists.txt create mode 100644 batch/cpp_application/README.md create mode 100644 batch/cpp_application/application.json create mode 100644 batch/cpp_application/application/CMakeLists.txt create mode 100644 batch/cpp_application/application/Dockerfile create mode 100644 batch/cpp_application/application/README.md create mode 100644 batch/cpp_application/application/src/main.cc create mode 100644 batch/cpp_application/main.cc create mode 100644 batch/cpp_application/vcpkg.json diff --git a/.gitignore b/.gitignore index 4cf80a97..ead26b5b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ cmake-build-debug/ .build/ .vscode/ cpp-samples-checkers/ +build/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 16a6a310..9fd125bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ include(ExternalProject) set(samples # cmake-format: sort + batch/cpp_application batch/simple bigquery/write cloud-run-hello-world diff --git a/batch/cpp_application/CMakeLists.txt b/batch/cpp_application/CMakeLists.txt new file mode 100644 index 00000000..e9ed7b64 --- /dev/null +++ b/batch/cpp_application/CMakeLists.txt @@ -0,0 +1,27 @@ +# ~~~ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +cmake_minimum_required(VERSION 3.20) + +# Define the project name and where to report bugs. +set(PACKAGE_BUGREPORT + "https://github.com/GoogleCloudPlatform/cpp-samples/issues") +project(cpp-samples-batch CXX) + +find_package(google_cloud_cpp_batch REQUIRED) + +add_executable(main main.cc) +target_link_libraries("main" PRIVATE google-cloud-cpp::batch) diff --git a/batch/cpp_application/README.md b/batch/cpp_application/README.md new file mode 100644 index 00000000..c852cb7d --- /dev/null +++ b/batch/cpp_application/README.md @@ -0,0 +1,124 @@ +# Using Cloud Batch + +This example shows how to run a C++ application on Cloud Batch job using C++. + +If you are not familiar with the Batch API, we recommend you first read the +[API overview] before starting this guide. + +## The example + +The following steps are included: + +1. Create a docker image +1. Upload it to Artifact registry +1. Create the job +1. Poll until the job finishes + +## Pre-reqs + +1. Install the [gcloud CLI](https://cloud.google.com/sdk/docs/install). +1. Install [docker](https://docs.docker.com/engine/install/). + +## 1. Create the docker image + +The instructions are [here](application/README.md). + +## 2. Upload it to Artifact registry + +1. \[If it does not already exist\] Create the artifact repository +1. Build the image locally +1. Tag and push the image to the artifact repository + +### 1. Create the artifact repository + +To run this example, replace the `[PROJECT ID]` placeholder with the id of your +project: + +Authorize via gcloud cli + +```shell +PROJECT_ID=[PROJECT_ID] +LOCATION="us-central1" +REPOSITORY="application-repo" + +gcloud auth login +gcloud config set project ${PROJECT_ID} +# Create the repository +gcloud artifacts repositories create ${REPOSITORY} \ + --repository-format=docker \ + --location=${LOCATION} \ + --description="Store the example C++ application" \ + --immutable-tags \ + --async +``` + +
+ To verify repo was created +``` +gcloud artifacts repositories list +``` + +You should see something like + +``` +application-repo DOCKER STANDARD_REPOSITORY Store the example C++ application us-central1 Google-managed key 2024-05-13T20:07:11 2024-05-13T20:07:11 0 +``` + +
+ +### 2. Build the image locally + +``` +cd batch/cpp_application/application +docker build --tag=application-image:latest . +``` + +### 3. Tag and push the image to the artifact repository + +``` +# Tag the image +docker tag application-image:latest ${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/application-image:latest + +# Push the image +docker push ${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/application-image:latest +``` + +## 3. Create the job + +## Compiling the Example + +This project uses `vcpkg` to install its dependencies. Clone `vcpkg` in your +`$HOME`: + +```shell +git clone -C $HOME https://github.com/microsoft/vcpkg.git +``` + +Install the typical development tools, on Ubuntu you would use: + +```shell +apt update && apt install -y build-essential cmake git ninja-build pkg-config g++ curl tar zip unzip +``` + +In this directory compile the dependencies and the code, this can take as long +as an hour, depending on the performance of your workstation: + +```shell +cd cpp-samples/batch/cpp_application +cmake -S . -B .build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$HOME/vcpkg/scripts/buildsystems/vcpkg.cmake +cmake --build .build +``` + +## Run the sample + +Run the example, replace the `[PROJECT ID]` placeholder with the id of your +project: + +```shell +.build/main [PROJECT ID] us-central1 cpp-application-run application.json application-repo +``` + +This submits the batch job and then polls until the job is complete. + +[api overview]: https://cloud.google.com/batch/docs diff --git a/batch/cpp_application/application.json b/batch/cpp_application/application.json new file mode 100644 index 00000000..7629df21 --- /dev/null +++ b/batch/cpp_application/application.json @@ -0,0 +1,36 @@ +{ + "taskGroups": [ + { + "taskSpec": { + "runnables": [ + { + "container": { + "imageUri": "", + } + } + ], + "computeResource": { + "cpuMilli": 2000, + "memoryMib": 16 + }, + "maxRetryCount": 2, + "maxRunDuration": "3600s" + }, + "taskCount": 1, + "parallelism": 1 + } + ], + "allocationPolicy": { + "instances": [ + { + "policy": { "machineType": "e2-standard-4" } + } + ] + }, + "labels": { + "env": "testing" + }, + "logsPolicy": { + "destination": "CLOUD_LOGGING" + } +} diff --git a/batch/cpp_application/application/CMakeLists.txt b/batch/cpp_application/application/CMakeLists.txt new file mode 100644 index 00000000..8a5855a3 --- /dev/null +++ b/batch/cpp_application/application/CMakeLists.txt @@ -0,0 +1,24 @@ +# ~~~ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +cmake_minimum_required(VERSION 3.20) + +# Define the project name and where to report bugs. +set(PACKAGE_BUGREPORT + "https://github.com/GoogleCloudPlatform/cpp-samples/issues") +project(cpp-samples-batch CXX) + +add_executable(main src/main.cc) diff --git a/batch/cpp_application/application/Dockerfile b/batch/cpp_application/application/Dockerfile new file mode 100644 index 00000000..3c8026c4 --- /dev/null +++ b/batch/cpp_application/application/Dockerfile @@ -0,0 +1,51 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# We chose Alpine to build the image because it has good support for creating +# statically-linked, small programs. +FROM alpine:3.19 AS build + +# Install the typical development tools for C++, and +# the base OS headers and libraries. +RUN apk update && \ + apk add \ + build-base \ + cmake \ + curl \ + git \ + gcc \ + g++ \ + libc-dev \ + linux-headers \ + ninja \ + pkgconfig \ + tar \ + unzip \ + zip + +# Copy the source code to /src and compile it. +COPY . /src +WORKDIR /src + +# Run the CMake configuration step, setting the options to create +# a statically linked C++ program +RUN cmake -S /src -B /build -GNinja \ + -DCMAKE_BUILD_TYPE=Release + +# Compile the binary and strip it to reduce its size. +RUN cmake --build /build +RUN strip /build/main + +# Make the program the entry point. +ENTRYPOINT [ "/build/main" ] diff --git a/batch/cpp_application/application/README.md b/batch/cpp_application/application/README.md new file mode 100644 index 00000000..3874f1ff --- /dev/null +++ b/batch/cpp_application/application/README.md @@ -0,0 +1,21 @@ +# To build and run + +``` +cmake -S . -B build +cmake --build build +build/main +``` + +# To create docker image + +``` +docker build --tag=application-image:latest . +``` + +## To run and enter your image + +``` +docker run -it --entrypoint bash application-image:latest +``` + +To exit container, type `exit` diff --git a/batch/cpp_application/application/src/main.cc b/batch/cpp_application/application/src/main.cc new file mode 100644 index 00000000..89db220a --- /dev/null +++ b/batch/cpp_application/application/src/main.cc @@ -0,0 +1,17 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +int main(int argc, char* argv[]) { std::cout << "Running my application\n"; } diff --git a/batch/cpp_application/main.cc b/batch/cpp_application/main.cc new file mode 100644 index 00000000..e95a295b --- /dev/null +++ b/batch/cpp_application/main.cc @@ -0,0 +1,145 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char* argv[]) try { + if (argc != 6) { + std::cerr << "Usage: " << argv[0] + << " " + "\n"; + return 1; + } + + namespace batch = ::google::cloud::batch_v1; + + std::string const project_id = argv[1]; + auto const location = google::cloud::Location(argv[1], argv[2]); + std::string const job_id = argv[3]; + std::string const job_file = argv[4]; + std::string const repository_name = argv[5]; + + // Parse the json and convert into protobuf format. + std::ifstream file(job_file, std::ios::in); + if (!file.is_open()) { + std::cout << "Failed to open JSON file: " << job_file << '\n'; + return 0; + } + auto contents = std::string{std::istreambuf_iterator(file), {}}; + google::cloud::batch::v1::Job job; + google::protobuf::util::JsonParseOptions options; + google::protobuf::util::Status status = + google::protobuf::util::JsonStringToMessage(contents, &job, options); + if (!status.ok()) throw status; + + // Modify the job for the containerized application + auto container = job.mutable_task_groups() + ->at(0) + .mutable_task_spec() + ->mutable_runnables() + ->at(0) + .mutable_container(); + std::string image_uri = location.location_id() + "-docker.pkg.dev/" + + location.project_id() + "/" + repository_name + + "/application-image:latest"; + container->set_image_uri(image_uri); + + // Create the cloud batch client. + auto client = batch::BatchServiceClient(batch::MakeBatchServiceConnection()); + + // Create a job. + auto response = client.CreateJob(location.FullName(), job, job_id); + + if (response.status().code() != google::cloud::StatusCode::kOk) { + if (response.status().code() == + google::cloud::StatusCode::kResourceExhausted) { + std::cout << "There already exists a job for the parent `" + << location.FullName() << "` and job_id: `" << job_id + << "`. Please try again with a new job id.\n"; + return 0; + } + throw std::move(response).status(); + } + + // On success, print the job. + std::cout << "Job : " << response->DebugString() << "\n"; + + // Poll the service using exponential backoff to check if job is ready and + // print once job is complete. + const auto kMinPollingInterval = std::chrono::minutes(2); + const auto kMaxPollingInterval = std::chrono::minutes(4); + const auto kMaxPollingTime = std::chrono::minutes(10); + + auto current_time = std::chrono::system_clock::now(); + auto in_time_t = std::chrono::system_clock::to_time_t(current_time); + std::cout << std::put_time(std::localtime(&in_time_t), "[%Y-%m-%d %X]") + << " Begin polling for job status\n"; + + const auto start_time = current_time; + auto delay = kMinPollingInterval; + while (current_time <= start_time + kMaxPollingTime) { + auto polling_response = + client.GetJob("projects/" + location.project_id() + "/locations/" + + location.location_id() + "/jobs/" + job_id); + if (polling_response.status().code() != google::cloud::StatusCode::kOk) { + throw std::move(polling_response).status(); + } + + switch (polling_response.value().status().state()) { + // 4 = google::cloud::batch::v1::JobStatus::State::SUCCEEDED + case 4: + std::cout << "Job succeeded!\n"; + return 0; + // 5 = google::cloud::batch::v1::JobStatus::State::FAILED + case 5: + std::cout << "Job failed!\n"; + return 0; + // 8 = google::cloud::batch::v1::JobStatus::State::CANCELLED + case 8: + std::cout << "Job cancelled!\n"; + return 0; + } + in_time_t = std::chrono::system_clock::to_time_t(current_time); + std::cout << std::put_time(std::localtime(&in_time_t), "[%Y-%m-%d %X]") + << " Job status: " + << google::cloud::batch::v1::JobStatus_State_Name( + polling_response.value().status().state()) + << "\n" + << "Current delay: " << delay.count() << " minute(s)\n"; + std::this_thread::sleep_for( + std::chrono::duration_cast(delay)); + delay = (std::min)(delay * 2, kMaxPollingInterval); + current_time = std::chrono::system_clock::now(); + } + in_time_t = + std::chrono::system_clock::to_time_t(start_time + kMaxPollingTime); + std::cout << std::put_time(std::localtime(&in_time_t), "[%Y-%m-%d %X]") + << " Max polling time passed\n"; + + return 0; +} catch (google::cloud::Status const& status) { + std::cerr << "google::cloud::Status thrown: " << status << "\n"; + return 1; +} catch (google::protobuf::util::Status const& status) { + std::cerr << "google::protobuf::util::Status thrown: " << status << "\n"; + return 1; +} diff --git a/batch/cpp_application/vcpkg.json b/batch/cpp_application/vcpkg.json new file mode 100644 index 00000000..7f84f41a --- /dev/null +++ b/batch/cpp_application/vcpkg.json @@ -0,0 +1,13 @@ +{ + "name": "gcp-cpp-samples-batch", + "version-string": "unversioned", + "homepage": "https://github.com/GoogleCloudPlatform/cpp-samples/", + "description": "An example using the Cloud Batch API", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": ["batch"] + } + ] + }