Skip to main content

Go

Apache OpenDAL™ Go Binding

Go Reference

opendal-go is a Native support Go binding without CGO enabled and is built on top of opendal-c.

go get github.com/apache/opendal/bindings/go@latest

opendal-go requires libffi to be installed.

Basic Usage

package main

import (
"fmt"
"github.com/apache/opendal-go-services/memory"
opendal "github.com/apache/opendal/bindings/go"
)

func main() {
// Initialize a new in-memory operator
op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{})
if err != nil {
panic(err)
}
defer op.Close()

// Write data to a file named "test"
err = op.Write("test", []byte("Hello opendal go binding!"))
if err != nil {
panic(err)
}

// Read data from the file "test"
data, err := op.Read("test")
if err != nil {
panic(err)
}
fmt.Printf("Read content: %s\n", data)

// List all entries under the root directory "/"
lister, err := op.List("/")
if err != nil {
panic(err)
}
defer lister.Close()

// Iterate through all entries
for lister.Next() {
entry := lister.Entry()

// Get entry name (not used in this example)
_ = entry.Name()

// Get metadata for the current entry
meta, _ := op.Stat(entry.Path())

// Print file size
fmt.Printf("Size: %d bytes\n", meta.ContentLength())

// Print last modified time
fmt.Printf("Last modified: %s\n", meta.LastModified())

// Check if the entry is a directory or a file
fmt.Printf("Is directory: %v, Is file: %v\n", meta.IsDir(), meta.IsFile())
}

// Check for any errors that occurred during iteration
if err := lister.Error(); err != nil {
panic(err)
}

// Copy a file
op.Copy("test", "test_copy")

// Rename a file
op.Rename("test", "test_rename")

// Delete a file
op.Delete("test_rename")
}

Run Tests

Behavior Tests

cd tests/behavior_tests
# Test a specific backend
export OPENDAL_TEST=memory
# Run all tests
CGO_ENABLE=0 go test -v -run TestBehavior
# Run specific test
CGO_ENABLE=0 go test -v -run TestBehavior/Write
# Run synchronously
CGO_ENABLE=0 GOMAXPROCS=1 go test -v -run TestBehavior

Benchmark

cd tests/behavior_tests
# Benchmark a specific backend
export OPENDAL_TEST=memory

go test -bench .
A benchmark between purego+libffi vs CGO

purego+libffi (as new.txt)

goos: linux
goarch: arm64
pkg: github.com/apache/opendal/bindings/go
BenchmarkWrite4KiB-10 1000000 2844 ns/op
BenchmarkWrite256KiB-10 163346 10092 ns/op
BenchmarkWrite4MiB-10 12900 99161 ns/op
BenchmarkWrite16MiB-10 1785 658210 ns/op
BenchmarkRead4KiB-10 194529 6387 ns/op
BenchmarkRead256KiB-10 14228 82704 ns/op
BenchmarkRead4MiB-10 981 1227872 ns/op
BenchmarkRead16MiB-10 328 3617185 ns/op
PASS
ok

CGO (as old.txt)

go test -bench=. -tags dynamic .
goos: linux
goarch: arm64
pkg: opendal.apache.org/go
BenchmarkWrite4KiB-10 241981 4240 ns/op
BenchmarkWrite256KiB-10 126464 10105 ns/op
BenchmarkWrite4MiB-10 13443 89578 ns/op
BenchmarkWrite16MiB-10 1737 646155 ns/op
BenchmarkRead4KiB-10 53535 20939 ns/op
BenchmarkRead256KiB-10 9008 132738 ns/op
BenchmarkRead4MiB-10 576 1846683 ns/op
BenchmarkRead16MiB-10 230 6305322 ns/op
PASS
ok

Diff with benchstat

benchstat old.txt new.txt
goos: linux
goarch: arm64
pkg: github.com/apache/opendal/bindings/go
│ new.txt │
│ sec/op │
Write4KiB-10 2.844µ ± ∞ ¹
Write256KiB-10 10.09µ ± ∞ ¹
Write4MiB-10 99.16µ ± ∞ ¹
Write16MiB-10 658.2µ ± ∞ ¹
Read4KiB-10 6.387µ ± ∞ ¹
Read256KiB-10 82.70µ ± ∞ ¹
Read4MiB-10 1.228m ± ∞ ¹
Read16MiB-10 3.617m ± ∞ ¹
geomean 90.23µ
¹ need >= 6 samples for confidence interval at level 0.95

pkg: opendal.apache.org/go
│ old.txt │
│ sec/op │
Write4KiB-10 4.240µ ± ∞ ¹
Write256KiB-10 10.11µ ± ∞ ¹
Write4MiB-10 89.58µ ± ∞ ¹
Write16MiB-10 646.2µ ± ∞ ¹
Read4KiB-10 20.94µ ± ∞ ¹
Read256KiB-10 132.7µ ± ∞ ¹
Read4MiB-10 1.847m ± ∞ ¹
Read16MiB-10 6.305m ± ∞ ¹
geomean 129.7µ
¹ need >= 6 samples for confidence interval at level 0.95

Capabilities

  • OperatorInfo
  • Stat
    • Metadata
  • IsExist
  • Read
    • Read
    • Reader -- implement as io.ReadCloser
  • Write
    • Write
    • Writer -- Need support from the C binding
  • Delete
  • CreateDir
  • Lister
    • Entry
    • Metadata -- Need support from the C binding
  • Copy
  • Rename

Development

The guide is based on Linux and Windows. For other platforms, please adjust the commands accordingly.

To develop the Go binding, you need to have the following dependencies installed:

  • zstd
  • Rust toolchain
  • Go
  • (Required for Windows) libffi-8.dll in the root of the workspace directory

We use go workspace to manage and build the dependencies. To set up the workspace, run the following commands:

For Linux

mkdir opendal_workspace
cd opendal_workspace
git clone --depth 1 git@github.com:apache/opendal.git
git clone --depth 1 git@github.com:apache/opendal-go-services.git

go work init
go work use ./opendal/bindings/go
go work use ./opendal/bindings/go/tests/behavior_tests
# use the backend you want to test, e.g., fs or memory
go work use ./opendal-go-services/fs
go work use ./opendal-go-services/memory

cat <<EOF > ./make_test.sh
#!/bin/bash

# Check if OPENDAL_TEST is set
if [ -z "\$OPENDAL_TEST" ]; then
echo "Error: OPENDAL_TEST environment variable is not set"
echo "Please set OPENDAL_TEST to specify which backend to test (e.g., fs or memory)"
exit 1
fi

# Specify the backend to test
export SERVICE="\$OPENDAL_TEST"

# Get architecture
architecture=\$(uname -m)
if [ "\$architecture" = "x86_64" ]; then
ARCH="amd64"
GOARCH="amd64"
elif [ "\$architecture" = "aarch64" ] || [ "\$architecture" = "arm64" ]; then
ARCH="arm64"
GOARCH="arm64"
else
ARCH="unknown"
fi

# Build opendal
cd opendal/bindings/c
cargo build
cd -

# Set environment variables
export GITHUB_WORKSPACE="\$PWD/opendal-go-services"
export VERSION="latest"
export TARGET="linux"
export DIR="\$GITHUB_WORKSPACE/libopendal_c_\${VERSION}_\${SERVICE}_\${TARGET}"

# Create directory if not exists
mkdir -p "\$DIR"

export OUTPUT="\$DIR/libopendal_c.\$TARGET.so.zst"
# Compress with zstd
zstd -19 opendal/bindings/c/target/debug/libopendal_c.so -o \$OUTPUT

# Set environment variables for test
export MATRIX='{"build": [{"target":"linux", "goos":"linux", "goarch": "'\$GOARCH'"}], "service": ["fs"]}'

# Generate code
cd opendal-go-services/internal/generate
go run generate.go
cd -

# Delete unnecessary files
rm -rf \$DIR

# Run tests
go test ./opendal/bindings/go/tests/behavior_tests -v -run TestBehavior
EOF

chmod +x ./make_test.sh

cd -

To build and run tests, run the following commands:

cd opendal_workspace

# specify the backend to test
export OPENDAL_TEST=fs
export OPENDAL_FS_ROOT=/tmp/opendal

# build the C binding and run the tests
./make_test.sh

cd -
For Windows

New-Item -ItemType Directory -Path opendal_workspace
Set-Location -Path opendal_workspace

git clone --depth 1 git@github.com:apache/opendal.git
git clone --depth 1 git@github.com:apache/opendal-go-services.git

go work init
go work use ./opendal/bindings/go
go work use ./opendal/bindings/go/tests/behavior_tests
# use the backend you want to test, e.g., fs or memory
go work use ./opendal-go-services/fs
go work use ./opendal-go-services/memory

@'
# Check if OPENDAL_TEST is set\;if (-not $env:OPENDAL_TEST) {\; Write-Error "OPENDAL_TEST environment variable is not set"\; Write-Host "Please set OPENDAL_TEST to specify which backend to test (e.g., fs or memory)"\; exit 1\;}\;# Specify the backend to test\;Set-Item -Path Env:SERVICE -Value "$env:OPENDAL_TEST"\;# Get architecture\;$architecture = (Get-WmiObject Win32_OperatingSystem).OSArchitecture\;\;if ($architecture -like "*64*") {\; $ARCH = "x86_64"\;} else {\; $ARCH = "unknown" \;}\;\;# Build opendal\;Push-Location opendal/bindings/c\;cargo build\;Pop-Location\;\;# Rename dll file\;Rename-Item opendal/bindings/c/target/debug/opendal_c.dll libopendal_c.dll\;\;# Set environment variables\;Set-Item -Path Env:GITHUB_WORKSPACE -Value "$PWD/opendal-go-services"\;Set-Item -Path Env:VERSION -Value "latest"\;Set-Item -Path Env:TARGET -Value "windows"\;Set-Item -Path Env:DIR -Value "$($env:GITHUB_WORKSPACE)/libopendal_c_$($env:VERSION)_$($env:SERVICE)_$($env:TARGET)"\;\;if (-not (Test-Path $env:DIR)) {\; New-Item -ItemType Directory -Path $env:DIR\;}\;\;# Compress with zstd\;zstd -19 opendal/bindings/c/target/debug/libopendal_c.dll -o "$($env:DIR)/libopendal_c.windows.dll.zst"\;\;Push-Location opendal-go-services/internal/generate\;go run generate.go\;Pop-Location\;# Remove Unnecessary files\;Remove-Item -Path $env:DIR -Recurse -Force\;# Set environment variables\;Set-Item -Path Env:MATRIX -Value '{"build": [{"target":"windows", "goos":"windows", "goarch": "amd64"}], "service": ["fs"]}'\;# Assume that libffi-8.dll is in the root of workspace directory\;Set-Item -Path Env:PATH -Value "$($env:PATH);$PWD"\;# Run tests\;go test ./opendal/bindings/go/tests/behavior_tests -v -run TestBehavior\;
'@ -replace "\\;","`n" | Out-File -FilePath "MakeTest.ps1" -Encoding UTF8

Pop-Location

To build and run tests, run the following commands:

Set-Location -Path opendal_workspace
# specify the backend to test
$env:OPENDAL_TEST = "fs"
$env:OPENDAL_FS_ROOT = $env:TEMP

# build the C binding and run the tests
.\MakeTest.ps1

Pop-Location

License and Trademarks

Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.