Building gRPC proto

This page shows how to build gRPC proto file for calling API to your Vald Cluster.

Vald provides the official client libraries (see: SDK document). If you can use one of the SDKs we recommend using it.

Target proto files

Vald defines the proto file for each API. Let’s check the below table for the details.

API service nameprotodependenciesusage
Insert vectors into Vald Agent
Update vectors stored in Vald Agent
Update vectors stored Vald Agent or Insert vectors into Vald Agent
Search similar vectors with query
Remove stored vectors from Vald Agent
Get object information of vector stored in Vald Agent

How to build protobuf

The way to build proto files

Let’s build proto files using your favorite programming language.

There are 3 steps to building API proto:

  1. Install gRPC tools

  2. Download Vald api proto files and external dependence

  3. Choose proto file(s) and build

Example: Build proto files in Rust

This section shows the example steps for building proto files using Rust.

There are many tools for building proto in Rust, we use tonic as an example.

  1. Check version

    This example runs in this environment.

    $ cargo version
    cargo 1.58.0 (7f08ace4f 2021-11-24)
    $ rustc -V
    rustc 1.58.0 (02072b482 2022-01-11)
    $ rustup -V
    rustup 1.24.3 (ce5817a94 2021-05-31)
    info: This is the version for the rustup toolchain manager, not the rustc compiler.
    info: The currently active `rustc` version is `rustc 1.57.0 (f1edd0429 2021-11-29)`
    $ rustup show
    Default host: x86_64-unknown-linux-gnu
    rustup home:  /home/user/.rustup
    stable-x86_64-unknown-linux-gnu (default)
    rustc 1.58.0 (02072b482 2022-01-11)
  2. Create project

    cargo new --lib vald-grpc
  3. Edit Cargo.toml

    cd vald-grpc && \
    vim Cargo.toml
    name = "vald-grpc"
    version = "0.1.0"
    edition = "2021"
    # See more keys and their definitions at
    name = "client"
    path = "src/"
    tonic = "0.6.2"
    tokio = "1.15"
    prost = "0.9"
    prost-types = "0.9"
    hdf5 = "0.8.1"
    chrono = "^0.4"
    tonic-build = "0.6.2"
  4. Download Vald proto files and dependence proto files

    // create proto file root dir
    mkdir -p proto
    // download vald proto files
    git clone \
    && cp -R vald/apis/proto/v1/vald proto/vald \
    && cp -R vald/apis/proto/v1/payload proto/payload \
    && rm -rf vald
    // download googleapis
    git clone \
    && cp -R googleapis/google proto/google \
    && rm -rf googleapis
    // download protoc-gen-validate
    git clone \
    && mv protoc-gen-validate proto/protoc-gen-validate
  5. Fixing import path

    find proto/vald -type f -name "*.proto" | xargs sed -i "s/apis\/proto\/v1\///g" && \
    find proto/vald -type f -name "*.proto" | xargs sed -i "s/github\.com\/googleapis\/googleapis\///g" && \
    find proto/payload -type f -name "*.proto" | xargs sed -i "s/github\.com\/googleapis\/googleapis\///g" && \
    find proto/payload -type f -name "*.proto" | xargs sed -i "s/github\.com\/envoyproxy\///g"
  6. Implement and Build proto


      fn main() -> Result<(), Box<dyn std::error::Error>> {
          let insert_proto = "./proto/vald/insert.proto";
          let update_proto = "./proto/vald/update.proto";
          let upsert_proto = "./proto/vald/upsert.proto";
          let search_proto = "./proto/vald/search.proto";
          let remove_proto = "./proto/vald/remove.proto";
          let object_proto = "./proto/vald/object.proto";
              .unwrap_or_else(|e| panic!("protobuf compile error: {}", e));
    2. build proto

      cargo build
  7. Edit Cargo.toml

    cd vald-grpc && \
    vim Cargo.toml
    name = "client"
    path = "src/"
  8. Implement code using client


      Import build proto in src/

      pub mod vald {
          pub mod v1 {
      pub mod payload {
          pub mod v1 {
      pub mod google {
          pub mod rpc {
          pub mod api {
          pub mod protobuf {
    2. src/

      There are 4 steps in src/

      1. Load dataset
      2. Insert vector to Vald cluster
      3. Search nearest neighbor vectors from Vald cluster after indexing finished
      4. Remove indexed vectors from Vald cluster

      The example is here:

      // import packages
      use chrono::Utc;
      use hdf5::{File, Result};
      use std::thread::sleep;
      use std::time;
      use tonic::transport::Endpoint;
      // import vald protos
      use vald_sample_rust_client::payload::v1::insert;
      use vald_sample_rust_client::payload::v1::object;
      use vald_sample_rust_client::payload::v1::remove;
      use vald_sample_rust_client::payload::v1::search;
      use vald_sample_rust_client::vald::v1::insert_client;
      use vald_sample_rust_client::vald::v1::remove_client;
      use vald_sample_rust_client::vald::v1::search_client;
      // Dataset file name
      static FILE: &str = "fashion-mnist-784-euclidean.hdf5";
      // Dataset name
      static DATASET: &str = "train";
      // set Vald cluster host
      static HOST: &str = "http://localhost:8080";
      // Time duration for waiting to finish `CreateIndex` and `SaveIndex`
      static DURATION: u64 = 15;
      // load data
      fn read_file() -> Result<Vec<Vec<f32>>> {
          let file = File::open(FILE).unwrap_or_else(|e| panic!("[ERR] failed to read file: {}", e));
          let data = file
              .unwrap_or_else(|e| panic!("[ERR] failed to get dataset: {}", e));
          let mut vector = Vec::new();
          for train in data.read_2d::<f32>()?.outer_iter() {
              let mut vec: Vec<f32> = Vec::new();
              vec.append(&mut train.to_vec());
              if vector.len() == 500 {
      #[tokio::main(flavor = "current_thread")]
      async fn main() -> Result<(), Box<dyn std::error::Error>> {
          print!("[Start] Load {} file\n", FILE);
          let vec = read_file()?;
          print!("[End] Success to load {} file\n", FILE);
          print!("[Start] Insert phase\n");
          // create insert client
          let mut insert_client =
          let mut ids: Vec<String> = Vec::new();
          for v in vec.iter() {
              let id = Utc::now().timestamp_nanos().to_string();
              // insert vector
              let _ = insert_client
                  .insert(insert::Request {
                      vector: Some(object::Vector {
                          id: id.to_string(),
                          vector: v.to_vec(),
                      config: Some(insert::Config {
                          skip_strict_exist_check: true,
                          filters: None,
                          timestamp: Utc::now().timestamp(),
          print!("[End] Finish Insert Phase\n");
          print!("[Sleep] Waiting SaveIndex is completed...\n");
          print!("[Start] Search phase\n");
          // create search client
          let mut search_client =
          for id in ids.clone() {
              // search nearest neighbor vectors using searchById method
              let res = search_client
                  .search_by_id(search::IdRequest {
                      id: id.to_string(),
                      config: Some(search::Config {
                          request_id: id.to_string(),
                          num: 10,
                          radius: -1.0,
                          epsilon: -1.0,
                          timeout: 500,
                          ingress_filters: None,
                          egress_filters: None,
              print!("[Id]: {:?}\n", id.to_string());
              print!("[Result]: {:#?}\n", res.into_inner().results);
          print!("[End] Finish Search Phase\n");
          print!("[Start] Remove phase\n");
          // create remove client
          let mut remove_client =
          for id in ids.clone() {
              // remove vectors
              let _ = remove_client
                  .remove(remove::Request {
                      id: Some(object::Id { id: id.to_string() }),
                      config: None,
          print!("[End] Finish Remove phase\n");
  9. Build

    Build implemented codes before running example code.

    cargo build
  10. Running example

    After creating Vald cluster, you can run an example code by the following command.

    cargo run src/