diff --git a/CMakeLists.txt b/CMakeLists.txt index 9fd125bb..75edbcb5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ include(ExternalProject) set(samples # cmake-format: sort batch/cpp_application + batch/parallel batch/simple bigquery/write cloud-run-hello-world diff --git a/batch/parallel/CMakeLists.txt b/batch/parallel/CMakeLists.txt new file mode 100644 index 00000000..e9ed7b64 --- /dev/null +++ b/batch/parallel/CMakeLists.txt @@ -0,0 +1,27 @@ +# ~~~ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +cmake_minimum_required(VERSION 3.20) + +# Define the project name and where to report bugs. +set(PACKAGE_BUGREPORT + "https://github.com/GoogleCloudPlatform/cpp-samples/issues") +project(cpp-samples-batch CXX) + +find_package(google_cloud_cpp_batch REQUIRED) + +add_executable(main main.cc) +target_link_libraries("main" PRIVATE google-cloud-cpp::batch) diff --git a/batch/parallel/README.md b/batch/parallel/README.md new file mode 100644 index 00000000..f50c497c --- /dev/null +++ b/batch/parallel/README.md @@ -0,0 +1,111 @@ +# Using Cloud Batch + +This example shows how to take an +[embarrasingly parallel](https://en.wikipedia.org/wiki/Embarrassingly_parallel) +job and run it on Cloud Batch job using C++. + +If you are not familiar with the Batch API, we recommend you first read the +[API overview] before starting this guide. + +## The example + +The following steps are included: + +1. Create a docker image +1. Upload it to Artifact registry +1. Create the Cloud Batch job + +## Pre-reqs + +1. Install the [gcloud CLI](https://cloud.google.com/sdk/docs/install). +1. Install [docker](https://docs.docker.com/engine/install/). + +## 1. Create the docker image + +The instructions are [here](application/README.md). + +## 2. Upload it to Artifact registry + +1. \[If it does not already exist\] Create the artifact repository +1. Build the image locally +1. Tag and push the image to the artifact repository + +### 1. Create the artifact repository + +To run this example, replace the `[PROJECT ID]` placeholder with the id of your +project: + +Authorize via gcloud cli + +```shell +PROJECT_ID=[PROJECT_ID] +LOCATION="us-central1" +REPOSITORY="parallel-repo" + +gcloud auth login +gcloud config set project ${PROJECT_ID} +# Create the repository +gcloud artifacts repositories create ${REPOSITORY} \ + --repository-format=docker \ + --location=${LOCATION} \ + --description="Store the example parallel C++ application" \ + --async +``` + +
+ To verify repo was created +``` +gcloud artifacts repositories list +``` + +You should see something like + +``` +parallel-repo DOCKER STANDARD_REPOSITORY Store the example parallel C++ application us-central1 Google-managed key 2024-05-21T12:39:54 2024-05-21T12:39:54 0 +``` + +
+ +### 2. Build the image locally + +``` +cd batch/parallel/application +docker build --tag=fimsim-image:latest . +``` + +### 3. Tag and push the image to the artifact repository + +``` +cd batch/parallel/application +gcloud builds submit --region=${LOCATION} --tag ${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/finsim-image:latest +``` + +## 3. Create the job using the gcloud CLI + +1. Replace the `imageURI` field in application.json + +``` + "runnables": [ + { + "container": { + "imageUri": "${LOCATION_ID}-docker.pkg.dev/${PROJECT_ID}/{REPOSITORY}/finsim-image:latest", + } + } + ], +``` + +2. Submit the job + +``` +gcloud batch jobs submit cpp-finsim-cli-run \ + --config=finsim.json \ + --location=us-central1 +``` + +3. Check on the job status + +``` +gcloud batch jobs describe cpp-finsim-cli-run --location=us-central1 +``` + +[api overview]: https://cloud.google.com/batch/docs diff --git a/batch/parallel/application/CMakeLists.txt b/batch/parallel/application/CMakeLists.txt new file mode 100644 index 00000000..c8b9f52c --- /dev/null +++ b/batch/parallel/application/CMakeLists.txt @@ -0,0 +1,24 @@ +# ~~~ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ~~~ + +cmake_minimum_required(VERSION 3.20) + +# Define the project name and where to report bugs. +set(PACKAGE_BUGREPORT + "https://github.com/GoogleCloudPlatform/cpp-samples/issues") +project(cpp-samples-batch CXX) + +add_executable(finsim src/finsim.cc) diff --git a/batch/parallel/application/Dockerfile b/batch/parallel/application/Dockerfile new file mode 100644 index 00000000..7f360786 --- /dev/null +++ b/batch/parallel/application/Dockerfile @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# We chose Alpine to build the image because it has good support for creating +# statically-linked, small programs. +FROM alpine:3.19 AS build + +# Install the typical development tools for C++, and +# the base OS headers and libraries. +RUN apk update && \ + apk add \ + build-base \ + cmake \ + curl \ + git \ + gcc \ + g++ \ + libc-dev \ + linux-headers \ + ninja \ + pkgconfig \ + tar \ + unzip \ + zip + +# Copy the source code to /src and compile it. +COPY . /src +WORKDIR /src + +# Run the CMake configuration step, setting the options to create +# a statically linked C++ program +RUN cmake -S /src -B /build -GNinja \ + -DCMAKE_BUILD_TYPE=Release + +# Compile the binary and strip it to reduce its size. +RUN cmake --build /build +RUN strip /build/finsim + +# Make the program the entry point. +ENTRYPOINT [ "/build/finsim" ] +CMD [ "input.txt"] diff --git a/batch/parallel/application/README.md b/batch/parallel/application/README.md new file mode 100644 index 00000000..afc81f69 --- /dev/null +++ b/batch/parallel/application/README.md @@ -0,0 +1,27 @@ +## The program + +This example runs a Monte Carlo financial simulation that approximates the +future value of a stock price given the inputs. + +# To build and run + +``` +cd cpp-samples/batch/parallel/application +cmake -S . -B build +cmake --build build +build/finsim input.txt +``` + +# To create docker image + +``` +docker build --tag=finsim-image:latest . +``` + +## To run and enter your image + +``` +docker run -it --entrypoint bash finsim-image:latest +``` + +To exit container, type `exit` diff --git a/batch/parallel/application/input.txt b/batch/parallel/application/input.txt new file mode 100644 index 00000000..7bcee1e4 --- /dev/null +++ b/batch/parallel/application/input.txt @@ -0,0 +1,6 @@ +1000 +1000 +GOOG +173.6 +.04 +.08 diff --git a/batch/parallel/application/src/finsim.cc b/batch/parallel/application/src/finsim.cc new file mode 100644 index 00000000..715f3357 --- /dev/null +++ b/batch/parallel/application/src/finsim.cc @@ -0,0 +1,113 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +struct InputConfig { + // Total simulations (e.g. 10 simulations). + std::int64_t simulations; + // Total iterations per simulation in days (e.g. 100 days). + std::int64_t iterations; + // Stock ticker name (e.g. GOOG). + std::string ticker; + // Initial stock price (e.g. $100). + double price; + // Daily percent variance (e.g. 5%) + double variance; + // Standard deviation in variance (e.g. 5% +/- 1%). + double deviation; + // Total threads used (max available using hardware). + std::int64_t total_threads; +}; + +void print_input_config(InputConfig const& config) { + std::cout << std::fixed << std::setprecision(2); + + std::cout << "\nInput Configuration\n"; + std::cout << "---------------------\n"; + std::cout << "Simulations: " << config.simulations << "\n"; + std::cout << "Iterations: " << config.iterations << "\n"; + std::cout << "Ticker: " << config.ticker << "\n"; + std::cout << "Price: $" << config.price << "\n"; + std::cout << "Variance: " << config.variance * 100.0 << " +/- " + << config.deviation * 100.0 << "%\n"; + std::cout << "Threads: " << config.total_threads << "\n"; +} + +double simulate(InputConfig const& config) { + double updated_price = config.price; + thread_local static std::mt19937 generator(std::random_device{}()); + std::uniform_real_distribution<> dis(-1 * config.deviation, config.deviation); + + for (int j = 0; j < config.iterations; ++j) { + updated_price = + updated_price * (1 + ((config.variance + dis(generator)) * 0.01)); + } + return updated_price; +} + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "usage: finsim \n"; + return 1; + } + + std::string input_filename = std::string(argv[1]); + + std::ifstream input_file; + input_file.open(input_filename); + std::vector lines; + if (input_file.is_open()) { + std::string line; + while (std::getline(input_file, line)) { + lines.push_back(line); + } + } else { + std::cout << "Couldn't open file\n"; + return 1; + } + + if (lines.size() != 6) { + std::cout << "Input is not the expected length\n"; + return 1; + } + + InputConfig input_config; + input_config.simulations = stoi(lines.at(0)); + input_config.iterations = stoi(lines.at(1)); + input_config.ticker = lines.at(2); + input_config.price = stof(lines.at(3)); + input_config.variance = stof(lines.at(4)); + input_config.deviation = stof(lines.at(5)); + + auto total_threads = + static_cast(std::thread::hardware_concurrency()); + input_config.total_threads = total_threads; + + print_input_config(input_config); + + std::vector results(input_config.simulations); + simulate(input_config); + +#pragma omp parallel for + for (int i = 0; i < input_config.simulations; i++) { + results[i] = simulate(input_config); + } + + double sum = 0.0; +#pragma omp parallel for reduction(+ : sum) + for (int i = 0; i < input_config.simulations; i++) { + sum += results[i]; + } + double mean = sum / input_config.simulations; + + std::cout << "\nOutput\n"; + std::cout << "---------------------\n"; + std::cout << "Mean final price: " << mean << "\n"; + + return 0; +} diff --git a/batch/parallel/finsim.json b/batch/parallel/finsim.json new file mode 100644 index 00000000..0dd962f3 --- /dev/null +++ b/batch/parallel/finsim.json @@ -0,0 +1,36 @@ +{ + "taskGroups": [ + { + "taskSpec": { + "runnables": [ + { + "container": { + "imageUri": "" + } + } + ], + "computeResource": { + "cpuMilli": 2000, + "memoryMib": 16 + }, + "maxRetryCount": 2, + "maxRunDuration": "3600s" + }, + "taskCount": 3, + "parallelism": 3 + } + ], + "allocationPolicy": { + "instances": [ + { + "policy": { "machineType": "e2-standard-4" } + } + ] + }, + "labels": { + "env": "testing" + }, + "logsPolicy": { + "destination": "CLOUD_LOGGING" + } +} diff --git a/batch/parallel/vcpkg.json b/batch/parallel/vcpkg.json new file mode 100644 index 00000000..7f84f41a --- /dev/null +++ b/batch/parallel/vcpkg.json @@ -0,0 +1,13 @@ +{ + "name": "gcp-cpp-samples-batch", + "version-string": "unversioned", + "homepage": "https://github.com/GoogleCloudPlatform/cpp-samples/", + "description": "An example using the Cloud Batch API", + "dependencies": [ + { + "name": "google-cloud-cpp", + "default-features": false, + "features": ["batch"] + } + ] + } diff --git a/gcs-fast-transfers/upload.cc b/gcs-fast-transfers/upload.cc index b22509f7..29a2c7ad 100644 --- a/gcs-fast-transfers/upload.cc +++ b/gcs-fast-transfers/upload.cc @@ -58,8 +58,7 @@ int main(int argc, char* argv[]) try { .value(); auto const end = std::chrono::steady_clock::now(); - std::cout << "DONE" - << "\n"; + std::cout << "DONE" << "\n"; std::cout << "The upload was successful, the object size is approximately " << format_size(metadata.size()) << "\n";