This commit is contained in:
robin
2026-03-13 14:25:13 +08:00
parent a25a474d6a
commit afbaaa869c
95 changed files with 4591 additions and 2578 deletions

View File

@@ -0,0 +1,178 @@
#!/usr/bin/env bash
set -euo pipefail
function host-goarch() {
case "$(uname -m)" in
x86_64|amd64)
echo "amd64"
;;
aarch64|arm64)
echo "arm64"
;;
*)
echo ""
;;
esac
}
function require-command() {
local command_name=$1
if ! command -v "$command_name" >/dev/null 2>&1; then
echo "missing required command: $command_name"
return 1
fi
}
function cpu_count() {
if command -v nproc >/dev/null 2>&1; then
nproc
else
getconf _NPROCESSORS_ONLN
fi
}
function download() {
local url=$1
local output=$2
if [ ! -f "$output" ]; then
curl -fsSL "$url" -o "$output"
fi
}
function prepare-dir() {
local dir=$1
rm -rf "$dir"
mkdir -p "$dir"
}
function copy-tree() {
local from_dir=$1
local to_dir=$2
rm -rf "$to_dir"
mkdir -p "$to_dir"
cp -a "$from_dir"/. "$to_dir"/
}
function copy-static-lib() {
local pattern=$1
local destination=$2
local source_path=""
source_path=$(find "$(dirname "$pattern")" -type f -name "$(basename "$pattern")" | head -n 1)
if [ -z "$source_path" ]; then
echo "missing static library: $pattern"
return 1
fi
cp "$source_path" "$destination"
}
ROOT=$(cd "$(dirname "$0")/.." && pwd)
ARCH=${1:-$(host-goarch)}
HOST_ARCH=$(host-goarch)
LIBPCAP_VERSION=${LIBPCAP_VERSION:-1.10.5}
BROTLI_VERSION=${BROTLI_VERSION:-1.1.0}
WORKDIR=${WORKDIR:-"$ROOT/.third_party_build"}
DOWNLOAD_DIR="$WORKDIR/downloads"
SOURCE_DIR="$WORKDIR/sources"
BUILD_DIR="$WORKDIR/build"
LIBPCAP_URL="https://www.tcpdump.org/release/libpcap-${LIBPCAP_VERSION}.tar.gz"
BROTLI_URL="https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz"
if [ -z "$ARCH" ]; then
echo "unsupported host architecture: $(uname -m)"
echo "usage: $0 [amd64|arm64]"
exit 1
fi
if [ "$ARCH" != "amd64" ] && [ "$ARCH" != "arm64" ]; then
echo "unsupported architecture: $ARCH"
echo "supported architectures: amd64 arm64"
exit 1
fi
if [ -n "$HOST_ARCH" ] && [ "$ARCH" != "$HOST_ARCH" ]; then
echo "this helper only builds native Ubuntu 22.04 static libraries"
echo "host arch: ${HOST_ARCH}, requested arch: ${ARCH}"
exit 1
fi
for tool in curl tar make cmake gcc g++ ar ranlib; do
require-command "$tool"
done
mkdir -p "$DOWNLOAD_DIR" "$SOURCE_DIR" "$BUILD_DIR"
echo "building third-party static libraries for ${ARCH} ..."
echo "workspace: $WORKDIR"
LIBPCAP_ARCHIVE="$DOWNLOAD_DIR/libpcap-${LIBPCAP_VERSION}.tar.gz"
BROTLI_ARCHIVE="$DOWNLOAD_DIR/brotli-${BROTLI_VERSION}.tar.gz"
download "$LIBPCAP_URL" "$LIBPCAP_ARCHIVE"
download "$BROTLI_URL" "$BROTLI_ARCHIVE"
LIBPCAP_SOURCE="$SOURCE_DIR/libpcap-${LIBPCAP_VERSION}"
BROTLI_SOURCE="$SOURCE_DIR/brotli-${BROTLI_VERSION}"
prepare-dir "$LIBPCAP_SOURCE"
prepare-dir "$BROTLI_SOURCE"
tar -xzf "$LIBPCAP_ARCHIVE" -C "$LIBPCAP_SOURCE" --strip-components=1
tar -xzf "$BROTLI_ARCHIVE" -C "$BROTLI_SOURCE" --strip-components=1
LIBPCAP_BUILD="$BUILD_DIR/libpcap-${ARCH}"
BROTLI_BUILD_SOURCE="$BUILD_DIR/brotli-source-${ARCH}"
BROTLI_BUILD_DIR="$BUILD_DIR/brotli-build-${ARCH}"
prepare-dir "$LIBPCAP_BUILD"
prepare-dir "$BROTLI_BUILD_SOURCE"
prepare-dir "$BROTLI_BUILD_DIR"
cp -a "$LIBPCAP_SOURCE"/. "$LIBPCAP_BUILD"/
cp -a "$BROTLI_SOURCE"/. "$BROTLI_BUILD_SOURCE"/
pushd "$LIBPCAP_BUILD" >/dev/null
./configure \
--disable-shared \
--enable-static \
--without-dbus \
--without-libnl \
--without-bluetooth \
--without-dag \
--without-septel \
--without-snf \
--without-turbocap \
--without-rdma \
--without-netmap \
CFLAGS="-O2 -fPIC"
make -j"$(cpu_count)"
popd >/dev/null
cmake -S "$BROTLI_BUILD_SOURCE" -B "$BROTLI_BUILD_DIR" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=OFF \
-DBROTLI_DISABLE_TESTS=ON \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
cmake --build "$BROTLI_BUILD_DIR" --parallel "$(cpu_count)"
mkdir -p "$ROOT/libs/libpcap/${ARCH}"
mkdir -p "$ROOT/libs/libbrotli/${ARCH}"
mkdir -p "$ROOT/libs/libpcap/src"
mkdir -p "$ROOT/libs/libbrotli/src"
cp "$LIBPCAP_BUILD/libpcap.a" "$ROOT/libs/libpcap/${ARCH}/libpcap.a"
copy-tree "$LIBPCAP_BUILD" "$ROOT/libs/libpcap/src/libpcap"
copy-static-lib "$BROTLI_BUILD_DIR/libbrotlicommon*.a" "$ROOT/libs/libbrotli/${ARCH}/libbrotlicommon.a"
copy-static-lib "$BROTLI_BUILD_DIR/libbrotlidec*.a" "$ROOT/libs/libbrotli/${ARCH}/libbrotlidec.a"
copy-static-lib "$BROTLI_BUILD_DIR/libbrotlienc*.a" "$ROOT/libs/libbrotli/${ARCH}/libbrotlienc.a"
copy-tree "$BROTLI_SOURCE" "$ROOT/libs/libbrotli/src/brotli"
echo "done"
echo "generated:"
echo " $ROOT/libs/libpcap/${ARCH}/libpcap.a"
echo " $ROOT/libs/libbrotli/${ARCH}/libbrotlicommon.a"
echo " $ROOT/libs/libbrotli/${ARCH}/libbrotlidec.a"
echo " $ROOT/libs/libbrotli/${ARCH}/libbrotlienc.a"
echo ""
echo "next:"
echo " cd \"$ROOT/build\" && ./build.sh linux ${ARCH} plus"

View File

@@ -1,42 +1,217 @@
#!/usr/bin/env bash
function find-binary() {
local candidate
for candidate in "$@"; do
if [ -z "$candidate" ]; then
continue
fi
if [ -x "$candidate" ]; then
echo "$candidate"
return 0
fi
if command -v "$candidate" >/dev/null 2>&1; then
command -v "$candidate"
return 0
fi
done
return 1
}
function host-goarch() {
case "$(uname -m)" in
x86_64|amd64)
echo "amd64"
;;
aarch64|arm64)
echo "arm64"
;;
i386|i486|i586|i686)
echo "386"
;;
mips64)
echo "mips64"
;;
mips64el)
echo "mips64le"
;;
*)
echo ""
;;
esac
}
function find-linux-static-toolchain() {
local arch=$1
local cc_bin=""
local cxx_bin=""
local host_arch
host_arch=$(host-goarch)
case "$arch" in
amd64)
cc_bin=$(find-binary \
"/usr/local/gcc/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-gcc" \
"/usr/local/opt/musl-cross/bin/x86_64-linux-musl-gcc" \
"x86_64-unknown-linux-gnu-gcc" \
"x86_64-linux-musl-gcc" \
"musl-gcc")
cxx_bin=$(find-binary \
"/usr/local/gcc/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-g++" \
"/usr/local/opt/musl-cross/bin/x86_64-linux-musl-g++" \
"x86_64-unknown-linux-gnu-g++" \
"x86_64-linux-musl-g++")
if [ -z "$cc_bin" ] && [ "$host_arch" = "amd64" ]; then
cc_bin=$(find-binary "gcc" "cc" "clang")
fi
if [ -z "$cxx_bin" ] && [ "$host_arch" = "amd64" ]; then
cxx_bin=$(find-binary "g++" "c++" "clang++")
fi
;;
386)
cc_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/i486-linux-musl-gcc" \
"i486-linux-musl-gcc")
cxx_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/i486-linux-musl-g++" \
"i486-linux-musl-g++")
if [ -z "$cc_bin" ] && [ "$host_arch" = "386" ]; then
cc_bin=$(find-binary "gcc" "cc" "clang")
fi
if [ -z "$cxx_bin" ] && [ "$host_arch" = "386" ]; then
cxx_bin=$(find-binary "g++" "c++" "clang++")
fi
;;
arm64)
cc_bin=$(find-binary \
"/usr/local/gcc/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc" \
"/usr/local/opt/musl-cross/bin/aarch64-linux-musl-gcc" \
"aarch64-unknown-linux-gnu-gcc" \
"aarch64-linux-musl-gcc")
cxx_bin=$(find-binary \
"/usr/local/gcc/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-g++" \
"/usr/local/opt/musl-cross/bin/aarch64-linux-musl-g++" \
"aarch64-unknown-linux-gnu-g++" \
"aarch64-linux-musl-g++")
if [ -z "$cc_bin" ] && [ "$host_arch" = "arm64" ]; then
cc_bin=$(find-binary "gcc" "cc" "clang")
fi
if [ -z "$cxx_bin" ] && [ "$host_arch" = "arm64" ]; then
cxx_bin=$(find-binary "g++" "c++" "clang++")
fi
;;
arm)
cc_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/arm-linux-musleabi-gcc" \
"arm-linux-musleabi-gcc")
cxx_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/arm-linux-musleabi-g++" \
"arm-linux-musleabi-g++")
if [ -z "$cc_bin" ] && [ "$host_arch" = "arm" ]; then
cc_bin=$(find-binary "gcc" "cc" "clang")
fi
if [ -z "$cxx_bin" ] && [ "$host_arch" = "arm" ]; then
cxx_bin=$(find-binary "g++" "c++" "clang++")
fi
;;
mips64)
cc_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/mips64-linux-musl-gcc" \
"mips64-linux-musl-gcc")
cxx_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/mips64-linux-musl-g++" \
"mips64-linux-musl-g++")
if [ -z "$cc_bin" ] && [ "$host_arch" = "mips64" ]; then
cc_bin=$(find-binary "gcc" "cc" "clang")
fi
if [ -z "$cxx_bin" ] && [ "$host_arch" = "mips64" ]; then
cxx_bin=$(find-binary "g++" "c++" "clang++")
fi
;;
mips64le)
cc_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/mips64el-linux-musl-gcc" \
"mips64el-linux-musl-gcc")
cxx_bin=$(find-binary \
"/usr/local/opt/musl-cross/bin/mips64el-linux-musl-g++" \
"mips64el-linux-musl-g++")
if [ -z "$cc_bin" ] && [ "$host_arch" = "mips64le" ]; then
cc_bin=$(find-binary "gcc" "cc" "clang")
fi
if [ -z "$cxx_bin" ] && [ "$host_arch" = "mips64le" ]; then
cxx_bin=$(find-binary "g++" "c++" "clang++")
fi
;;
*)
return 1
;;
esac
if [ -z "$cc_bin" ]; then
return 1
fi
if [ -z "$cxx_bin" ]; then
cxx_bin="$cc_bin"
fi
echo "$cc_bin|$cxx_bin"
return 0
}
function require-packet-static-libs() {
local srcdir=$1
local arch=$2
local missing=0
local file
for file in \
"${srcdir}/libs/libpcap/${arch}/libpcap.a" \
"${srcdir}/libs/libbrotli/${arch}/libbrotlienc.a" \
"${srcdir}/libs/libbrotli/${arch}/libbrotlidec.a" \
"${srcdir}/libs/libbrotli/${arch}/libbrotlicommon.a"; do
if [ ! -f "$file" ]; then
echo "missing required plus packet library: $file"
missing=1
fi
done
if [ "$missing" -ne 0 ]; then
return 1
fi
return 0
}
function build() {
ROOT=$(dirname $0)
ROOT=$(dirname "$0")
NAME="edge-node"
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
DIST=$ROOT/"../dist/${NAME}"
MUSL_DIR="/usr/local/opt/musl-cross/bin"
SRCDIR=$(realpath "$ROOT/..")
# for macOS users: precompiled gcc can be downloaded from https://github.com/messense/homebrew-macos-cross-toolchains
GCC_X86_64_DIR="/usr/local/gcc/x86_64-unknown-linux-gnu/bin"
GCC_ARM64_DIR="/usr/local/gcc/aarch64-unknown-linux-gnu/bin"
OS=${1}
ARCH=${2}
TAG=${3}
if [ -z "$OS" ]; then
echo "usage: build.sh OS ARCH"
exit
exit 1
fi
if [ -z "$ARCH" ]; then
echo "usage: build.sh OS ARCH"
exit
exit 1
fi
if [ -z "$TAG" ]; then
TAG="community"
fi
echo "checking ..."
ZIP_PATH=$(which zip)
if [ -z "$ZIP_PATH" ]; then
if ! command -v zip >/dev/null 2>&1; then
echo "we need 'zip' command to compress files"
exit
exit 1
fi
echo "building v${VERSION}/${OS}/${ARCH}/${TAG} ..."
# 生成 zip 文件名时不包含 plus 标记
if [ "${TAG}" = "plus" ]; then
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
else
@@ -63,11 +238,8 @@ function build() {
cp -R "$ROOT"/pages "$DIST"/
copy_fluent_bit_assets "$ROOT" "$DIST" "$OS" "$ARCH" || exit 1
# we support TOA on linux only
if [ "$OS" == "linux" ] && [ -f "${ROOT}/edge-toa/edge-toa-${ARCH}" ]
then
if [ ! -d "$DIST/edge-toa" ]
then
if [ "$OS" == "linux" ] && [ -f "${ROOT}/edge-toa/edge-toa-${ARCH}" ]; then
if [ ! -d "$DIST/edge-toa" ]; then
mkdir "$DIST/edge-toa"
fi
cp "${ROOT}/edge-toa/edge-toa-${ARCH}" "$DIST/edge-toa/edge-toa"
@@ -75,96 +247,59 @@ function build() {
echo "building ..."
CC_PATH=""
CXX_PATH=""
CC_BIN=""
CXX_BIN=""
CGO_LDFLAGS=""
CGO_CFLAGS=""
BUILD_TAG=$TAG
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
if [ "${ARCH}" == "amd64" ]; then
# build with script support
if [ -d $GCC_X86_64_DIR ]; then
MUSL_DIR=$GCC_X86_64_DIR
CC_PATH="x86_64-unknown-linux-gnu-gcc"
CXX_PATH="x86_64-unknown-linux-gnu-g++"
if [ "$TAG" = "plus" ]; then
BUILD_TAG="plus,script,packet"
fi
else
CC_PATH="x86_64-linux-musl-gcc"
CXX_PATH="x86_64-linux-musl-g++"
fi
fi
if [ "${ARCH}" == "386" ]; then
CC_PATH="i486-linux-musl-gcc"
CXX_PATH="i486-linux-musl-g++"
fi
if [ "${ARCH}" == "arm64" ]; then
# build with script support
if [ -d $GCC_ARM64_DIR ]; then
MUSL_DIR=$GCC_ARM64_DIR
CC_PATH="aarch64-unknown-linux-gnu-gcc"
CXX_PATH="aarch64-unknown-linux-gnu-g++"
if [ "$TAG" = "plus" ]; then
BUILD_TAG="plus,script,packet"
fi
else
CC_PATH="aarch64-linux-musl-gcc"
CXX_PATH="aarch64-linux-musl-g++"
fi
fi
if [ "${ARCH}" == "arm" ]; then
CC_PATH="arm-linux-musleabi-gcc"
CXX_PATH="arm-linux-musleabi-g++"
fi
if [ "${ARCH}" == "mips64" ]; then
CC_PATH="mips64-linux-musl-gcc"
CXX_PATH="mips64-linux-musl-g++"
fi
if [ "${ARCH}" == "mips64le" ]; then
CC_PATH="mips64el-linux-musl-gcc"
CXX_PATH="mips64el-linux-musl-g++"
fi
fi
# libpcap
if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
require-packet-static-libs "$SRCDIR" "$ARCH" || exit 1
BUILD_TAG="plus,script,packet"
CGO_LDFLAGS="-L${SRCDIR}/libs/libpcap/${ARCH} -lpcap -L${SRCDIR}/libs/libbrotli/${ARCH} -lbrotlienc -lbrotlidec -lbrotlicommon"
CGO_CFLAGS="-I${SRCDIR}/libs/libpcap/src/libpcap -I${SRCDIR}/libs/libpcap/src/libpcap/pcap -I${SRCDIR}/libs/libbrotli/src/brotli/c/include -I${SRCDIR}/libs/libbrotli/src/brotli/c/include/brotli"
fi
if [ ! -z $CC_PATH ]; then
env CC=$MUSL_DIR/$CC_PATH \
CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" \
GOARCH="${ARCH}" CGO_ENABLED=1 \
CGO_LDFLAGS="${CGO_LDFLAGS}" \
CGO_CFLAGS="${CGO_CFLAGS}" \
go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
else
if [[ `uname` == *"Linux"* ]] && [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
BUILD_TAG="plus,script,packet"
if [ "$OS" == "linux" ]; then
TOOLCHAIN=$(find-linux-static-toolchain "$ARCH")
if [ -z "$TOOLCHAIN" ]; then
echo "could not find a static Linux toolchain for ${ARCH}"
echo "install a musl cross compiler or provide /usr/local/gcc toolchains before building"
exit 1
fi
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
CC_BIN=${TOOLCHAIN%|*}
CXX_BIN=${TOOLCHAIN#*|}
env CC="$CC_BIN" \
CXX="$CXX_BIN" \
GOOS="${OS}" \
GOARCH="${ARCH}" \
CGO_ENABLED=1 \
CGO_LDFLAGS="${CGO_LDFLAGS}" \
CGO_CFLAGS="${CGO_CFLAGS}" \
go build -trimpath -tags "$BUILD_TAG" -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
else
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" \
go build -trimpath -tags "$BUILD_TAG" -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
fi
if [ ! -f "${DIST}/bin/${NAME}" ]; then
echo "build failed!"
exit
exit 1
fi
# delete hidden files
find "$DIST" -name ".DS_Store" -delete
find "$DIST" -name ".gitignore" -delete
echo "zip files"
cd "${DIST}/../" || exit
cd "${DIST}/../" || exit 1
if [ -f "${ZIP}" ]; then
rm -f "${ZIP}"
fi
zip -r -X -q "${ZIP}" ${NAME}/
rm -rf ${NAME}
cd - || exit
cd - || exit 1
echo "OK"
}
@@ -253,7 +388,7 @@ function lookup-version() {
echo "$VERSION"
else
echo "could not match version"
exit
exit 1
fi
}

View File

@@ -1 +1 @@
{"speed":1,"speedMB":1400,"countTests":3}
{"speed":1,"speedMB":1510,"countTests":10}

View File

@@ -1,3 +1,3 @@
#!/usr/bin/env bash
sudo go run -tags="plus script packet" ../cmd/edge-node/main.go
sudo go run -tags="plus script" ../cmd/edge-node/main.go