Go
Apache OpenDAL™ Go Binding
opendal-go is a Native support Go binding without CGO enabled and is built on top of opendal-c.
go get github.com/apache/opendal/bindings/go@latest
opendal-go requires libffi to be installed.
Basic Usage
package main
import (
"fmt"
"github.com/apache/opendal-go-services/memory"
opendal "github.com/apache/opendal/bindings/go"
)
func main() {
// Initialize a new in-memory operator
op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{})
if err != nil {
panic(err)
}
defer op.Close()
// Write data to a file named "test"
err = op.Write("test", []byte("Hello opendal go binding!"))
if err != nil {
panic(err)
}
// Read data from the file "test"
data, err := op.Read("test")
if err != nil {
panic(err)
}
fmt.Printf("Read content: %s\n", data)
// List all entries under the root directory "/"
lister, err := op.List("/")
if err != nil {
panic(err)
}
defer lister.Close()
// Iterate through all entries
for lister.Next() {
entry := lister.Entry()
// Get entry name (not used in this example)
_ = entry.Name()
// Get metadata for the current entry
meta, _ := op.Stat(entry.Path())
// Print file size
fmt.Printf("Size: %d bytes\n", meta.ContentLength())
// Print last modified time
fmt.Printf("Last modified: %s\n", meta.LastModified())
// Check if the entry is a directory or a file
fmt.Printf("Is directory: %v, Is file: %v\n", meta.IsDir(), meta.IsFile())
}
// Check for any errors that occurred during iteration
if err := lister.Error(); err != nil {
panic(err)
}
// Copy a file
op.Copy("test", "test_copy")
// Rename a file
op.Rename("test", "test_rename")
// Delete a file
op.Delete("test_rename")
}
Run Tests
Behavior Tests
cd tests/behavior_tests
# Test a specific backend
export OPENDAL_TEST=memory
# Run all tests
CGO_ENABLE=0 go test -v -run TestBehavior
# Run specific test
CGO_ENABLE=0 go test -v -run TestBehavior/Write
# Run synchronously
CGO_ENABLE=0 GOMAXPROCS=1 go test -v -run TestBehavior
Benchmark
cd tests/behavior_tests
# Benchmark a specific backend
export OPENDAL_TEST=memory
go test -bench .
A benchmark between purego+libffi vs CGO
purego+libffi (as new.txt
)
goos: linux
goarch: arm64
pkg: github.com/apache/opendal/bindings/go
BenchmarkWrite4KiB-10 1000000 2844 ns/op
BenchmarkWrite256KiB-10 163346 10092 ns/op
BenchmarkWrite4MiB-10 12900 99161 ns/op
BenchmarkWrite16MiB-10 1785 658210 ns/op
BenchmarkRead4KiB-10 194529 6387 ns/op
BenchmarkRead256KiB-10 14228 82704 ns/op
BenchmarkRead4MiB-10 981 1227872 ns/op
BenchmarkRead16MiB-10 328 3617185 ns/op
PASS
ok
CGO (as old.txt
)
go test -bench=. -tags dynamic .
goos: linux
goarch: arm64
pkg: opendal.apache.org/go
BenchmarkWrite4KiB-10 241981 4240 ns/op
BenchmarkWrite256KiB-10 126464 10105 ns/op
BenchmarkWrite4MiB-10 13443 89578 ns/op
BenchmarkWrite16MiB-10 1737 646155 ns/op
BenchmarkRead4KiB-10 53535 20939 ns/op
BenchmarkRead256KiB-10 9008 132738 ns/op
BenchmarkRead4MiB-10 576 1846683 ns/op
BenchmarkRead16MiB-10 230 6305322 ns/op
PASS
ok
Diff with benchstat
benchstat old.txt new.txt
goos: linux
goarch: arm64
pkg: github.com/apache/opendal/bindings/go
│ new.txt │
│ sec/op │
Write4KiB-10 2.844µ ± ∞ ¹
Write256KiB-10 10.09µ ± ∞ ¹
Write4MiB-10 99.16µ ± ∞ ¹
Write16MiB-10 658.2µ ± ∞ ¹
Read4KiB-10 6.387µ ± ∞ ¹
Read256KiB-10 82.70µ ± ∞ ¹
Read4MiB-10 1.228m ± ∞ ¹
Read16MiB-10 3.617m ± ∞ ¹
geomean 90.23µ
¹ need >= 6 samples for confidence interval at level 0.95
pkg: opendal.apache.org/go
│ old.txt │
│ sec/op │
Write4KiB-10 4.240µ ± ∞ ¹
Write256KiB-10 10.11µ ± ∞ ¹
Write4MiB-10 89.58µ ± ∞ ¹
Write16MiB-10 646.2µ ± ∞ ¹
Read4KiB-10 20.94µ ± ∞ ¹
Read256KiB-10 132.7µ ± ∞ ¹
Read4MiB-10 1.847m ± ∞ ¹
Read16MiB-10 6.305m ± ∞ ¹
geomean 129.7µ
¹ need >= 6 samples for confidence interval at level 0.95
Capabilities
- OperatorInfo
- Stat
- Metadata
- IsExist
- Read
- Read
- Reader -- implement as
io.ReadCloser
- Write
- Write
- Writer -- Need support from the C binding
- Delete
- CreateDir
- Lister
- Entry
- Metadata -- Need support from the C binding
- Copy
- Rename
Development
The guide is based on Linux and Windows. For other platforms, please adjust the commands accordingly.
To develop the Go binding, you need to have the following dependencies installed:
- zstd
- Rust toolchain
- Go
- (Required for Windows) libffi-8.dll in the root of the workspace directory
We use go workspace
to manage and build the dependencies. To set up the workspace, run the following commands:
For Linux
mkdir opendal_workspace
cd opendal_workspace
git clone --depth 1 git@github.com:apache/opendal.git
git clone --depth 1 git@github.com:apache/opendal-go-services.git
go work init
go work use ./opendal/bindings/go
go work use ./opendal/bindings/go/tests/behavior_tests
# use the backend you want to test, e.g., fs or memory
go work use ./opendal-go-services/fs
go work use ./opendal-go-services/memory
cat <<EOF > ./make_test.sh
#!/bin/bash
# Check if OPENDAL_TEST is set
if [ -z "\$OPENDAL_TEST" ]; then
echo "Error: OPENDAL_TEST environment variable is not set"
echo "Please set OPENDAL_TEST to specify which backend to test (e.g., fs or memory)"
exit 1
fi
# Specify the backend to test
export SERVICE="\$OPENDAL_TEST"
# Get architecture
architecture=\$(uname -m)
if [ "\$architecture" = "x86_64" ]; then
ARCH="amd64"
GOARCH="amd64"
elif [ "\$architecture" = "aarch64" ] || [ "\$architecture" = "arm64" ]; then
ARCH="arm64"
GOARCH="arm64"
else
ARCH="unknown"
fi
# Build opendal
cd opendal/bindings/c
cargo build
cd -
# Set environment variables
export GITHUB_WORKSPACE="\$PWD/opendal-go-services"
export VERSION="latest"
export TARGET="linux"
export DIR="\$GITHUB_WORKSPACE/libopendal_c_\${VERSION}_\${SERVICE}_\${TARGET}"
# Create directory if not exists
mkdir -p "\$DIR"
export OUTPUT="\$DIR/libopendal_c.\$TARGET.so.zst"
# Compress with zstd
zstd -19 opendal/bindings/c/target/debug/libopendal_c.so -o \$OUTPUT
# Set environment variables for test
export MATRIX='{"build": [{"target":"linux", "goos":"linux", "goarch": "'\$GOARCH'"}], "service": ["fs"]}'
# Generate code
cd opendal-go-services/internal/generate
go run generate.go
cd -
# Delete unnecessary files
rm -rf \$DIR
# Run tests
go test ./opendal/bindings/go/tests/behavior_tests -v -run TestBehavior
EOF
chmod +x ./make_test.sh
cd -
To build and run tests, run the following commands:
cd opendal_workspace
# specify the backend to test
export OPENDAL_TEST=fs
export OPENDAL_FS_ROOT=/tmp/opendal
# build the C binding and run the tests
./make_test.sh
cd -
For Windows
New-Item -ItemType Directory -Path opendal_workspace
Set-Location -Path opendal_workspace
git clone --depth 1 git@github.com:apache/opendal.git
git clone --depth 1 git@github.com:apache/opendal-go-services.git
go work init
go work use ./opendal/bindings/go
go work use ./opendal/bindings/go/tests/behavior_tests
# use the backend you want to test, e.g., fs or memory
go work use ./opendal-go-services/fs
go work use ./opendal-go-services/memory
@'
# Check if OPENDAL_TEST is set\;if (-not $env:OPENDAL_TEST) {\; Write-Error "OPENDAL_TEST environment variable is not set"\; Write-Host "Please set OPENDAL_TEST to specify which backend to test (e.g., fs or memory)"\; exit 1\;}\;# Specify the backend to test\;Set-Item -Path Env:SERVICE -Value "$env:OPENDAL_TEST"\;# Get architecture\;$architecture = (Get-WmiObject Win32_OperatingSystem).OSArchitecture\;\;if ($architecture -like "*64*") {\; $ARCH = "x86_64"\;} else {\; $ARCH = "unknown" \;}\;\;# Build opendal\;Push-Location opendal/bindings/c\;cargo build\;Pop-Location\;\;# Rename dll file\;Rename-Item opendal/bindings/c/target/debug/opendal_c.dll libopendal_c.dll\;\;# Set environment variables\;Set-Item -Path Env:GITHUB_WORKSPACE -Value "$PWD/opendal-go-services"\;Set-Item -Path Env:VERSION -Value "latest"\;Set-Item -Path Env:TARGET -Value "windows"\;Set-Item -Path Env:DIR -Value "$($env:GITHUB_WORKSPACE)/libopendal_c_$($env:VERSION)_$($env:SERVICE)_$($env:TARGET)"\;\;if (-not (Test-Path $env:DIR)) {\; New-Item -ItemType Directory -Path $env:DIR\;}\;\;# Compress with zstd\;zstd -19 opendal/bindings/c/target/debug/libopendal_c.dll -o "$($env:DIR)/libopendal_c.windows.dll.zst"\;\;Push-Location opendal-go-services/internal/generate\;go run generate.go\;Pop-Location\;# Remove Unnecessary files\;Remove-Item -Path $env:DIR -Recurse -Force\;# Set environment variables\;Set-Item -Path Env:MATRIX -Value '{"build": [{"target":"windows", "goos":"windows", "goarch": "amd64"}], "service": ["fs"]}'\;# Assume that libffi-8.dll is in the root of workspace directory\;Set-Item -Path Env:PATH -Value "$($env:PATH);$PWD"\;# Run tests\;go test ./opendal/bindings/go/tests/behavior_tests -v -run TestBehavior\;
'@ -replace "\\;","`n" | Out-File -FilePath "MakeTest.ps1" -Encoding UTF8
Pop-Location
To build and run tests, run the following commands:
Set-Location -Path opendal_workspace
# specify the backend to test
$env:OPENDAL_TEST = "fs"
$env:OPENDAL_FS_ROOT = $env:TEMP
# build the C binding and run the tests
.\MakeTest.ps1
Pop-Location
License and Trademarks
Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.