1.5.0
This commit is contained in:
2
EdgeNode/.gitignore
vendored
2
EdgeNode/.gitignore
vendored
@@ -8,6 +8,8 @@ configs/api_node.yaml
|
||||
|
||||
# Build binaries
|
||||
bin/
|
||||
libs/
|
||||
.third_party_build/
|
||||
|
||||
# Runtime Data
|
||||
data/
|
||||
|
||||
178
EdgeNode/build/build-libs-ubuntu2204.sh
Executable file
178
EdgeNode/build/build-libs-ubuntu2204.sh
Executable file
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
function host-goarch() {
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64)
|
||||
echo "amd64"
|
||||
;;
|
||||
aarch64|arm64)
|
||||
echo "arm64"
|
||||
;;
|
||||
*)
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function require-command() {
|
||||
local command_name=$1
|
||||
if ! command -v "$command_name" >/dev/null 2>&1; then
|
||||
echo "missing required command: $command_name"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function cpu_count() {
|
||||
if command -v nproc >/dev/null 2>&1; then
|
||||
nproc
|
||||
else
|
||||
getconf _NPROCESSORS_ONLN
|
||||
fi
|
||||
}
|
||||
|
||||
function download() {
|
||||
local url=$1
|
||||
local output=$2
|
||||
if [ ! -f "$output" ]; then
|
||||
curl -fsSL "$url" -o "$output"
|
||||
fi
|
||||
}
|
||||
|
||||
function prepare-dir() {
|
||||
local dir=$1
|
||||
rm -rf "$dir"
|
||||
mkdir -p "$dir"
|
||||
}
|
||||
|
||||
function copy-tree() {
|
||||
local from_dir=$1
|
||||
local to_dir=$2
|
||||
rm -rf "$to_dir"
|
||||
mkdir -p "$to_dir"
|
||||
cp -a "$from_dir"/. "$to_dir"/
|
||||
}
|
||||
|
||||
function copy-static-lib() {
|
||||
local pattern=$1
|
||||
local destination=$2
|
||||
local source_path=""
|
||||
|
||||
source_path=$(find "$(dirname "$pattern")" -type f -name "$(basename "$pattern")" | head -n 1)
|
||||
if [ -z "$source_path" ]; then
|
||||
echo "missing static library: $pattern"
|
||||
return 1
|
||||
fi
|
||||
|
||||
cp "$source_path" "$destination"
|
||||
}
|
||||
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
ARCH=${1:-$(host-goarch)}
|
||||
HOST_ARCH=$(host-goarch)
|
||||
LIBPCAP_VERSION=${LIBPCAP_VERSION:-1.10.5}
|
||||
BROTLI_VERSION=${BROTLI_VERSION:-1.1.0}
|
||||
WORKDIR=${WORKDIR:-"$ROOT/.third_party_build"}
|
||||
DOWNLOAD_DIR="$WORKDIR/downloads"
|
||||
SOURCE_DIR="$WORKDIR/sources"
|
||||
BUILD_DIR="$WORKDIR/build"
|
||||
LIBPCAP_URL="https://www.tcpdump.org/release/libpcap-${LIBPCAP_VERSION}.tar.gz"
|
||||
BROTLI_URL="https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz"
|
||||
|
||||
if [ -z "$ARCH" ]; then
|
||||
echo "unsupported host architecture: $(uname -m)"
|
||||
echo "usage: $0 [amd64|arm64]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$ARCH" != "amd64" ] && [ "$ARCH" != "arm64" ]; then
|
||||
echo "unsupported architecture: $ARCH"
|
||||
echo "supported architectures: amd64 arm64"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$HOST_ARCH" ] && [ "$ARCH" != "$HOST_ARCH" ]; then
|
||||
echo "this helper only builds native Ubuntu 22.04 static libraries"
|
||||
echo "host arch: ${HOST_ARCH}, requested arch: ${ARCH}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for tool in curl tar make cmake gcc g++ ar ranlib; do
|
||||
require-command "$tool"
|
||||
done
|
||||
|
||||
mkdir -p "$DOWNLOAD_DIR" "$SOURCE_DIR" "$BUILD_DIR"
|
||||
|
||||
echo "building third-party static libraries for ${ARCH} ..."
|
||||
echo "workspace: $WORKDIR"
|
||||
|
||||
LIBPCAP_ARCHIVE="$DOWNLOAD_DIR/libpcap-${LIBPCAP_VERSION}.tar.gz"
|
||||
BROTLI_ARCHIVE="$DOWNLOAD_DIR/brotli-${BROTLI_VERSION}.tar.gz"
|
||||
|
||||
download "$LIBPCAP_URL" "$LIBPCAP_ARCHIVE"
|
||||
download "$BROTLI_URL" "$BROTLI_ARCHIVE"
|
||||
|
||||
LIBPCAP_SOURCE="$SOURCE_DIR/libpcap-${LIBPCAP_VERSION}"
|
||||
BROTLI_SOURCE="$SOURCE_DIR/brotli-${BROTLI_VERSION}"
|
||||
prepare-dir "$LIBPCAP_SOURCE"
|
||||
prepare-dir "$BROTLI_SOURCE"
|
||||
|
||||
tar -xzf "$LIBPCAP_ARCHIVE" -C "$LIBPCAP_SOURCE" --strip-components=1
|
||||
tar -xzf "$BROTLI_ARCHIVE" -C "$BROTLI_SOURCE" --strip-components=1
|
||||
|
||||
LIBPCAP_BUILD="$BUILD_DIR/libpcap-${ARCH}"
|
||||
BROTLI_BUILD_SOURCE="$BUILD_DIR/brotli-source-${ARCH}"
|
||||
BROTLI_BUILD_DIR="$BUILD_DIR/brotli-build-${ARCH}"
|
||||
prepare-dir "$LIBPCAP_BUILD"
|
||||
prepare-dir "$BROTLI_BUILD_SOURCE"
|
||||
prepare-dir "$BROTLI_BUILD_DIR"
|
||||
|
||||
cp -a "$LIBPCAP_SOURCE"/. "$LIBPCAP_BUILD"/
|
||||
cp -a "$BROTLI_SOURCE"/. "$BROTLI_BUILD_SOURCE"/
|
||||
|
||||
pushd "$LIBPCAP_BUILD" >/dev/null
|
||||
./configure \
|
||||
--disable-shared \
|
||||
--enable-static \
|
||||
--without-dbus \
|
||||
--without-libnl \
|
||||
--without-bluetooth \
|
||||
--without-dag \
|
||||
--without-septel \
|
||||
--without-snf \
|
||||
--without-turbocap \
|
||||
--without-rdma \
|
||||
--without-netmap \
|
||||
CFLAGS="-O2 -fPIC"
|
||||
make -j"$(cpu_count)"
|
||||
popd >/dev/null
|
||||
|
||||
cmake -S "$BROTLI_BUILD_SOURCE" -B "$BROTLI_BUILD_DIR" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DBUILD_SHARED_LIBS=OFF \
|
||||
-DBROTLI_DISABLE_TESTS=ON \
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||
cmake --build "$BROTLI_BUILD_DIR" --parallel "$(cpu_count)"
|
||||
|
||||
mkdir -p "$ROOT/libs/libpcap/${ARCH}"
|
||||
mkdir -p "$ROOT/libs/libbrotli/${ARCH}"
|
||||
mkdir -p "$ROOT/libs/libpcap/src"
|
||||
mkdir -p "$ROOT/libs/libbrotli/src"
|
||||
|
||||
cp "$LIBPCAP_BUILD/libpcap.a" "$ROOT/libs/libpcap/${ARCH}/libpcap.a"
|
||||
copy-tree "$LIBPCAP_BUILD" "$ROOT/libs/libpcap/src/libpcap"
|
||||
|
||||
copy-static-lib "$BROTLI_BUILD_DIR/libbrotlicommon*.a" "$ROOT/libs/libbrotli/${ARCH}/libbrotlicommon.a"
|
||||
copy-static-lib "$BROTLI_BUILD_DIR/libbrotlidec*.a" "$ROOT/libs/libbrotli/${ARCH}/libbrotlidec.a"
|
||||
copy-static-lib "$BROTLI_BUILD_DIR/libbrotlienc*.a" "$ROOT/libs/libbrotli/${ARCH}/libbrotlienc.a"
|
||||
copy-tree "$BROTLI_SOURCE" "$ROOT/libs/libbrotli/src/brotli"
|
||||
|
||||
echo "done"
|
||||
echo "generated:"
|
||||
echo " $ROOT/libs/libpcap/${ARCH}/libpcap.a"
|
||||
echo " $ROOT/libs/libbrotli/${ARCH}/libbrotlicommon.a"
|
||||
echo " $ROOT/libs/libbrotli/${ARCH}/libbrotlidec.a"
|
||||
echo " $ROOT/libs/libbrotli/${ARCH}/libbrotlienc.a"
|
||||
echo ""
|
||||
echo "next:"
|
||||
echo " cd \"$ROOT/build\" && ./build.sh linux ${ARCH} plus"
|
||||
@@ -1,42 +1,217 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
function find-binary() {
|
||||
local candidate
|
||||
for candidate in "$@"; do
|
||||
if [ -z "$candidate" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ -x "$candidate" ]; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
if command -v "$candidate" >/dev/null 2>&1; then
|
||||
command -v "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
function host-goarch() {
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64)
|
||||
echo "amd64"
|
||||
;;
|
||||
aarch64|arm64)
|
||||
echo "arm64"
|
||||
;;
|
||||
i386|i486|i586|i686)
|
||||
echo "386"
|
||||
;;
|
||||
mips64)
|
||||
echo "mips64"
|
||||
;;
|
||||
mips64el)
|
||||
echo "mips64le"
|
||||
;;
|
||||
*)
|
||||
echo ""
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function find-linux-static-toolchain() {
|
||||
local arch=$1
|
||||
local cc_bin=""
|
||||
local cxx_bin=""
|
||||
local host_arch
|
||||
|
||||
host_arch=$(host-goarch)
|
||||
|
||||
case "$arch" in
|
||||
amd64)
|
||||
cc_bin=$(find-binary \
|
||||
"/usr/local/gcc/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-gcc" \
|
||||
"/usr/local/opt/musl-cross/bin/x86_64-linux-musl-gcc" \
|
||||
"x86_64-unknown-linux-gnu-gcc" \
|
||||
"x86_64-linux-musl-gcc" \
|
||||
"musl-gcc")
|
||||
cxx_bin=$(find-binary \
|
||||
"/usr/local/gcc/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-g++" \
|
||||
"/usr/local/opt/musl-cross/bin/x86_64-linux-musl-g++" \
|
||||
"x86_64-unknown-linux-gnu-g++" \
|
||||
"x86_64-linux-musl-g++")
|
||||
if [ -z "$cc_bin" ] && [ "$host_arch" = "amd64" ]; then
|
||||
cc_bin=$(find-binary "gcc" "cc" "clang")
|
||||
fi
|
||||
if [ -z "$cxx_bin" ] && [ "$host_arch" = "amd64" ]; then
|
||||
cxx_bin=$(find-binary "g++" "c++" "clang++")
|
||||
fi
|
||||
;;
|
||||
386)
|
||||
cc_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/i486-linux-musl-gcc" \
|
||||
"i486-linux-musl-gcc")
|
||||
cxx_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/i486-linux-musl-g++" \
|
||||
"i486-linux-musl-g++")
|
||||
if [ -z "$cc_bin" ] && [ "$host_arch" = "386" ]; then
|
||||
cc_bin=$(find-binary "gcc" "cc" "clang")
|
||||
fi
|
||||
if [ -z "$cxx_bin" ] && [ "$host_arch" = "386" ]; then
|
||||
cxx_bin=$(find-binary "g++" "c++" "clang++")
|
||||
fi
|
||||
;;
|
||||
arm64)
|
||||
cc_bin=$(find-binary \
|
||||
"/usr/local/gcc/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc" \
|
||||
"/usr/local/opt/musl-cross/bin/aarch64-linux-musl-gcc" \
|
||||
"aarch64-unknown-linux-gnu-gcc" \
|
||||
"aarch64-linux-musl-gcc")
|
||||
cxx_bin=$(find-binary \
|
||||
"/usr/local/gcc/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-g++" \
|
||||
"/usr/local/opt/musl-cross/bin/aarch64-linux-musl-g++" \
|
||||
"aarch64-unknown-linux-gnu-g++" \
|
||||
"aarch64-linux-musl-g++")
|
||||
if [ -z "$cc_bin" ] && [ "$host_arch" = "arm64" ]; then
|
||||
cc_bin=$(find-binary "gcc" "cc" "clang")
|
||||
fi
|
||||
if [ -z "$cxx_bin" ] && [ "$host_arch" = "arm64" ]; then
|
||||
cxx_bin=$(find-binary "g++" "c++" "clang++")
|
||||
fi
|
||||
;;
|
||||
arm)
|
||||
cc_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/arm-linux-musleabi-gcc" \
|
||||
"arm-linux-musleabi-gcc")
|
||||
cxx_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/arm-linux-musleabi-g++" \
|
||||
"arm-linux-musleabi-g++")
|
||||
if [ -z "$cc_bin" ] && [ "$host_arch" = "arm" ]; then
|
||||
cc_bin=$(find-binary "gcc" "cc" "clang")
|
||||
fi
|
||||
if [ -z "$cxx_bin" ] && [ "$host_arch" = "arm" ]; then
|
||||
cxx_bin=$(find-binary "g++" "c++" "clang++")
|
||||
fi
|
||||
;;
|
||||
mips64)
|
||||
cc_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/mips64-linux-musl-gcc" \
|
||||
"mips64-linux-musl-gcc")
|
||||
cxx_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/mips64-linux-musl-g++" \
|
||||
"mips64-linux-musl-g++")
|
||||
if [ -z "$cc_bin" ] && [ "$host_arch" = "mips64" ]; then
|
||||
cc_bin=$(find-binary "gcc" "cc" "clang")
|
||||
fi
|
||||
if [ -z "$cxx_bin" ] && [ "$host_arch" = "mips64" ]; then
|
||||
cxx_bin=$(find-binary "g++" "c++" "clang++")
|
||||
fi
|
||||
;;
|
||||
mips64le)
|
||||
cc_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/mips64el-linux-musl-gcc" \
|
||||
"mips64el-linux-musl-gcc")
|
||||
cxx_bin=$(find-binary \
|
||||
"/usr/local/opt/musl-cross/bin/mips64el-linux-musl-g++" \
|
||||
"mips64el-linux-musl-g++")
|
||||
if [ -z "$cc_bin" ] && [ "$host_arch" = "mips64le" ]; then
|
||||
cc_bin=$(find-binary "gcc" "cc" "clang")
|
||||
fi
|
||||
if [ -z "$cxx_bin" ] && [ "$host_arch" = "mips64le" ]; then
|
||||
cxx_bin=$(find-binary "g++" "c++" "clang++")
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "$cc_bin" ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ -z "$cxx_bin" ]; then
|
||||
cxx_bin="$cc_bin"
|
||||
fi
|
||||
|
||||
echo "$cc_bin|$cxx_bin"
|
||||
return 0
|
||||
}
|
||||
|
||||
function require-packet-static-libs() {
|
||||
local srcdir=$1
|
||||
local arch=$2
|
||||
local missing=0
|
||||
local file
|
||||
|
||||
for file in \
|
||||
"${srcdir}/libs/libpcap/${arch}/libpcap.a" \
|
||||
"${srcdir}/libs/libbrotli/${arch}/libbrotlienc.a" \
|
||||
"${srcdir}/libs/libbrotli/${arch}/libbrotlidec.a" \
|
||||
"${srcdir}/libs/libbrotli/${arch}/libbrotlicommon.a"; do
|
||||
if [ ! -f "$file" ]; then
|
||||
echo "missing required plus packet library: $file"
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$missing" -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
function build() {
|
||||
ROOT=$(dirname $0)
|
||||
ROOT=$(dirname "$0")
|
||||
NAME="edge-node"
|
||||
VERSION=$(lookup-version "$ROOT"/../internal/const/const.go)
|
||||
DIST=$ROOT/"../dist/${NAME}"
|
||||
MUSL_DIR="/usr/local/opt/musl-cross/bin"
|
||||
SRCDIR=$(realpath "$ROOT/..")
|
||||
|
||||
# for macOS users: precompiled gcc can be downloaded from https://github.com/messense/homebrew-macos-cross-toolchains
|
||||
GCC_X86_64_DIR="/usr/local/gcc/x86_64-unknown-linux-gnu/bin"
|
||||
GCC_ARM64_DIR="/usr/local/gcc/aarch64-unknown-linux-gnu/bin"
|
||||
|
||||
OS=${1}
|
||||
ARCH=${2}
|
||||
TAG=${3}
|
||||
|
||||
if [ -z "$OS" ]; then
|
||||
echo "usage: build.sh OS ARCH"
|
||||
exit
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$ARCH" ]; then
|
||||
echo "usage: build.sh OS ARCH"
|
||||
exit
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$TAG" ]; then
|
||||
TAG="community"
|
||||
fi
|
||||
|
||||
echo "checking ..."
|
||||
ZIP_PATH=$(which zip)
|
||||
if [ -z "$ZIP_PATH" ]; then
|
||||
if ! command -v zip >/dev/null 2>&1; then
|
||||
echo "we need 'zip' command to compress files"
|
||||
exit
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "building v${VERSION}/${OS}/${ARCH}/${TAG} ..."
|
||||
# 生成 zip 文件名时不包含 plus 标记
|
||||
if [ "${TAG}" = "plus" ]; then
|
||||
ZIP="${NAME}-${OS}-${ARCH}-v${VERSION}.zip"
|
||||
else
|
||||
@@ -63,11 +238,8 @@ function build() {
|
||||
cp -R "$ROOT"/pages "$DIST"/
|
||||
copy_fluent_bit_assets "$ROOT" "$DIST" "$OS" "$ARCH" || exit 1
|
||||
|
||||
# we support TOA on linux only
|
||||
if [ "$OS" == "linux" ] && [ -f "${ROOT}/edge-toa/edge-toa-${ARCH}" ]
|
||||
then
|
||||
if [ ! -d "$DIST/edge-toa" ]
|
||||
then
|
||||
if [ "$OS" == "linux" ] && [ -f "${ROOT}/edge-toa/edge-toa-${ARCH}" ]; then
|
||||
if [ ! -d "$DIST/edge-toa" ]; then
|
||||
mkdir "$DIST/edge-toa"
|
||||
fi
|
||||
cp "${ROOT}/edge-toa/edge-toa-${ARCH}" "$DIST/edge-toa/edge-toa"
|
||||
@@ -75,96 +247,59 @@ function build() {
|
||||
|
||||
echo "building ..."
|
||||
|
||||
CC_PATH=""
|
||||
CXX_PATH=""
|
||||
CC_BIN=""
|
||||
CXX_BIN=""
|
||||
CGO_LDFLAGS=""
|
||||
CGO_CFLAGS=""
|
||||
BUILD_TAG=$TAG
|
||||
if [[ `uname -a` == *"Darwin"* && "${OS}" == "linux" ]]; then
|
||||
if [ "${ARCH}" == "amd64" ]; then
|
||||
# build with script support
|
||||
if [ -d $GCC_X86_64_DIR ]; then
|
||||
MUSL_DIR=$GCC_X86_64_DIR
|
||||
CC_PATH="x86_64-unknown-linux-gnu-gcc"
|
||||
CXX_PATH="x86_64-unknown-linux-gnu-g++"
|
||||
if [ "$TAG" = "plus" ]; then
|
||||
BUILD_TAG="plus,script,packet"
|
||||
fi
|
||||
else
|
||||
CC_PATH="x86_64-linux-musl-gcc"
|
||||
CXX_PATH="x86_64-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
if [ "${ARCH}" == "386" ]; then
|
||||
CC_PATH="i486-linux-musl-gcc"
|
||||
CXX_PATH="i486-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "arm64" ]; then
|
||||
# build with script support
|
||||
if [ -d $GCC_ARM64_DIR ]; then
|
||||
MUSL_DIR=$GCC_ARM64_DIR
|
||||
CC_PATH="aarch64-unknown-linux-gnu-gcc"
|
||||
CXX_PATH="aarch64-unknown-linux-gnu-g++"
|
||||
if [ "$TAG" = "plus" ]; then
|
||||
BUILD_TAG="plus,script,packet"
|
||||
fi
|
||||
else
|
||||
CC_PATH="aarch64-linux-musl-gcc"
|
||||
CXX_PATH="aarch64-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
if [ "${ARCH}" == "arm" ]; then
|
||||
CC_PATH="arm-linux-musleabi-gcc"
|
||||
CXX_PATH="arm-linux-musleabi-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "mips64" ]; then
|
||||
CC_PATH="mips64-linux-musl-gcc"
|
||||
CXX_PATH="mips64-linux-musl-g++"
|
||||
fi
|
||||
if [ "${ARCH}" == "mips64le" ]; then
|
||||
CC_PATH="mips64el-linux-musl-gcc"
|
||||
CXX_PATH="mips64el-linux-musl-g++"
|
||||
fi
|
||||
fi
|
||||
|
||||
# libpcap
|
||||
if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
|
||||
if [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
|
||||
require-packet-static-libs "$SRCDIR" "$ARCH" || exit 1
|
||||
BUILD_TAG="plus,script,packet"
|
||||
CGO_LDFLAGS="-L${SRCDIR}/libs/libpcap/${ARCH} -lpcap -L${SRCDIR}/libs/libbrotli/${ARCH} -lbrotlienc -lbrotlidec -lbrotlicommon"
|
||||
CGO_CFLAGS="-I${SRCDIR}/libs/libpcap/src/libpcap -I${SRCDIR}/libs/libpcap/src/libpcap/pcap -I${SRCDIR}/libs/libbrotli/src/brotli/c/include -I${SRCDIR}/libs/libbrotli/src/brotli/c/include/brotli"
|
||||
fi
|
||||
|
||||
if [ ! -z $CC_PATH ]; then
|
||||
env CC=$MUSL_DIR/$CC_PATH \
|
||||
CXX=$MUSL_DIR/$CXX_PATH GOOS="${OS}" \
|
||||
GOARCH="${ARCH}" CGO_ENABLED=1 \
|
||||
CGO_LDFLAGS="${CGO_LDFLAGS}" \
|
||||
CGO_CFLAGS="${CGO_CFLAGS}" \
|
||||
go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
else
|
||||
if [[ `uname` == *"Linux"* ]] && [ "$OS" == "linux" ] && [[ "$ARCH" == "amd64" || "$ARCH" == "arm64" ]] && [ "$TAG" == "plus" ]; then
|
||||
BUILD_TAG="plus,script,packet"
|
||||
if [ "$OS" == "linux" ]; then
|
||||
TOOLCHAIN=$(find-linux-static-toolchain "$ARCH")
|
||||
if [ -z "$TOOLCHAIN" ]; then
|
||||
echo "could not find a static Linux toolchain for ${ARCH}"
|
||||
echo "install a musl cross compiler or provide /usr/local/gcc toolchains before building"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" go build -trimpath -tags $BUILD_TAG -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
CC_BIN=${TOOLCHAIN%|*}
|
||||
CXX_BIN=${TOOLCHAIN#*|}
|
||||
|
||||
env CC="$CC_BIN" \
|
||||
CXX="$CXX_BIN" \
|
||||
GOOS="${OS}" \
|
||||
GOARCH="${ARCH}" \
|
||||
CGO_ENABLED=1 \
|
||||
CGO_LDFLAGS="${CGO_LDFLAGS}" \
|
||||
CGO_CFLAGS="${CGO_CFLAGS}" \
|
||||
go build -trimpath -tags "$BUILD_TAG" -o "$DIST"/bin/${NAME} -ldflags "-linkmode external -extldflags -static -s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
else
|
||||
env GOOS="${OS}" GOARCH="${ARCH}" CGO_ENABLED=1 CGO_LDFLAGS="${CGO_LDFLAGS}" CGO_CFLAGS="${CGO_CFLAGS}" \
|
||||
go build -trimpath -tags "$BUILD_TAG" -o "$DIST"/bin/${NAME} -ldflags="-s -w" "$ROOT"/../cmd/edge-node/main.go
|
||||
fi
|
||||
|
||||
if [ ! -f "${DIST}/bin/${NAME}" ]; then
|
||||
echo "build failed!"
|
||||
exit
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# delete hidden files
|
||||
find "$DIST" -name ".DS_Store" -delete
|
||||
find "$DIST" -name ".gitignore" -delete
|
||||
|
||||
echo "zip files"
|
||||
cd "${DIST}/../" || exit
|
||||
cd "${DIST}/../" || exit 1
|
||||
if [ -f "${ZIP}" ]; then
|
||||
rm -f "${ZIP}"
|
||||
fi
|
||||
zip -r -X -q "${ZIP}" ${NAME}/
|
||||
rm -rf ${NAME}
|
||||
cd - || exit
|
||||
cd - || exit 1
|
||||
|
||||
echo "OK"
|
||||
}
|
||||
@@ -253,7 +388,7 @@ function lookup-version() {
|
||||
echo "$VERSION"
|
||||
else
|
||||
echo "could not match version"
|
||||
exit
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"speed":1,"speedMB":1400,"countTests":3}
|
||||
{"speed":1,"speedMB":1510,"countTests":10}
|
||||
@@ -1,3 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
sudo go run -tags="plus script packet" ../cmd/edge-node/main.go
|
||||
sudo go run -tags="plus script" ../cmd/edge-node/main.go
|
||||
|
||||
@@ -20,7 +20,6 @@ require (
|
||||
github.com/dchest/captcha v0.0.0-00010101000000-000000000000
|
||||
github.com/fsnotify/fsnotify v1.7.0
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/google/nftables v0.2.0
|
||||
github.com/huaweicloud/huaweicloud-sdk-go-obs v3.24.9+incompatible
|
||||
github.com/iwind/TeaGo v0.0.0-20240411075713-6c1fc9aca7b6
|
||||
|
||||
@@ -67,8 +67,6 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
|
||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||
github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8=
|
||||
github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4=
|
||||
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package teaconst
|
||||
|
||||
const (
|
||||
Version = "1.4.9" //1.3.8.2
|
||||
Version = "1.5.0" //1.3.8.2
|
||||
|
||||
ProductName = "Edge Node"
|
||||
ProcessName = "edge-node"
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package networksecurity
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
teaconst "github.com/TeaOSLab/EdgeNode/internal/const"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/events"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/monitor"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/goman"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/netpackets"
|
||||
"github.com/iwind/TeaGo/Tea"
|
||||
"github.com/iwind/TeaGo/maps"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
var SharedManager = NewManager()
|
||||
|
||||
func init() {
|
||||
if !teaconst.IsMain {
|
||||
return
|
||||
}
|
||||
|
||||
events.On(events.EventLoaded, func() {
|
||||
nodeConfig, _ := nodeconfigs.SharedNodeConfig()
|
||||
if nodeConfig != nil {
|
||||
go SharedManager.Apply(nodeConfig.NetworkSecurityPolicy)
|
||||
}
|
||||
})
|
||||
|
||||
events.On(events.EventQuit, func() {
|
||||
go SharedManager.Apply(nil)
|
||||
})
|
||||
|
||||
goman.New(func() {
|
||||
var ticker = time.NewTicker(1 * time.Minute)
|
||||
for range ticker.C {
|
||||
SharedManager.Upload()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
listener *netpackets.Listener
|
||||
isRunning bool
|
||||
|
||||
policy *nodeconfigs.NetworkSecurityPolicy
|
||||
|
||||
totalTCPPacketsMinutely uint64
|
||||
totalUDPPacketsMinutely uint64
|
||||
totalICMPPacketsMinutely uint64
|
||||
}
|
||||
|
||||
func NewManager() *Manager {
|
||||
return &Manager{}
|
||||
}
|
||||
|
||||
// Apply 应用配置
|
||||
// 非线程安全
|
||||
func (this *Manager) Apply(policy *nodeconfigs.NetworkSecurityPolicy) {
|
||||
if this.policy != nil && this.policy.IsSame(policy) {
|
||||
return
|
||||
}
|
||||
|
||||
this.policy = policy
|
||||
|
||||
if policy == nil ||
|
||||
policy.Status == nodeconfigs.NetworkSecurityStatusOff ||
|
||||
(policy.Status == nodeconfigs.NetworkSecurityStatusAuto && runtime.NumCPU() < 8) {
|
||||
if this.listener != nil {
|
||||
remotelogs.Println("NETWORK_SECURITY_MANAGER", "stop")
|
||||
this.listener.Stop()
|
||||
}
|
||||
this.isRunning = false
|
||||
return
|
||||
}
|
||||
|
||||
if this.listener == nil {
|
||||
this.listener = netpackets.NewListener()
|
||||
|
||||
// References:
|
||||
// - https://biot.com/capstats/bpf.html
|
||||
// - https://www.ibm.com/docs/en/qsip/7.4?topic=queries-berkeley-packet-filters
|
||||
// - https://www.tcpdump.org/manpages/tcpdump.1.html
|
||||
|
||||
if Tea.IsTesting() || utils.IsDebugEnv() { // dev environment
|
||||
this.listener.SetBPF("(tcp or udp or icmp) and not net 127 and not net ::1")
|
||||
} else {
|
||||
this.listener.SetBPF("(tcp or udp or icmp) and not src net 127 and not src net 192.168 and not src net 172.16 and not src net ::1 and not src net 10")
|
||||
}
|
||||
this.listener.AddFilter(this)
|
||||
}
|
||||
|
||||
if !this.isRunning {
|
||||
this.isRunning = true
|
||||
remotelogs.Println("NETWORK_SECURITY_MANAGER", "start")
|
||||
err := this.listener.Start() // long run function
|
||||
if err != nil {
|
||||
remotelogs.Error("NETWORK_SECURITY_MANAGER", "start listener failed: "+err.Error())
|
||||
}
|
||||
this.isRunning = false
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Manager) FilterMeta(meta *netpackets.PacketMeta) {
|
||||
switch meta.LayerType {
|
||||
case netpackets.LayerTypeTCP:
|
||||
// 这里不需要试用atomic,因为数据不需要那么精确
|
||||
this.totalTCPPacketsMinutely++
|
||||
case netpackets.LayerTypeUDP:
|
||||
this.totalUDPPacketsMinutely++
|
||||
case netpackets.LayerTypeICMPv4, netpackets.LayerTypeICMPv6:
|
||||
this.totalICMPPacketsMinutely++
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Manager) Upload() {
|
||||
if !this.isRunning {
|
||||
return
|
||||
}
|
||||
|
||||
monitor.SharedValueQueue.Add(nodeconfigs.NodeValueItemNetworkPackets, maps.Map{
|
||||
"tcpInPPS": this.totalTCPPacketsMinutely / 60,
|
||||
"udpInPPS": this.totalUDPPacketsMinutely / 60,
|
||||
"icmpInPPS": this.totalICMPPacketsMinutely / 60,
|
||||
})
|
||||
|
||||
this.totalTCPPacketsMinutely = 0
|
||||
this.totalUDPPacketsMinutely = 0
|
||||
this.totalICMPPacketsMinutely = 0
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package networksecurity_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
networksecurity "github.com/TeaOSLab/EdgeNode/internal/network-security"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestManager_Apply(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
if os.Getgid() > 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var manager = networksecurity.NewManager()
|
||||
var policy = nodeconfigs.NewNetworkSecurityPolicy()
|
||||
manager.Apply(policy)
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
sudo go test -v -tags="plus packet" -run '^TestManager_Apply'
|
||||
@@ -9,14 +9,20 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// 执行认证
|
||||
// 鎵ц璁よ瘉
|
||||
func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
if this.web.Auth == nil || !this.web.Auth.IsOn {
|
||||
return
|
||||
}
|
||||
|
||||
for _, ref := range this.web.Auth.PolicyRefs {
|
||||
if !ref.IsOn || ref.AuthPolicy == nil || !ref.AuthPolicy.IsOn {
|
||||
if !ref.IsOn {
|
||||
continue
|
||||
}
|
||||
if ref.AuthPolicy == nil {
|
||||
continue
|
||||
}
|
||||
if !ref.AuthPolicy.IsOn {
|
||||
continue
|
||||
}
|
||||
if !ref.AuthPolicy.MatchRequest(this.RawReq) {
|
||||
@@ -36,7 +42,7 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
return writer.StatusCode(), nil
|
||||
}, this.Format)
|
||||
if err != nil {
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "认证策略执行失败", false)
|
||||
this.write50x(err, http.StatusInternalServerError, "Failed to execute the AuthPolicy", "璁よ瘉绛栫暐鎵ц澶辫触", false)
|
||||
return
|
||||
}
|
||||
if ok {
|
||||
@@ -45,28 +51,28 @@ func (this *HTTPRequest) doAuth() (shouldStop bool) {
|
||||
}
|
||||
this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
|
||||
return
|
||||
} else {
|
||||
// Basic Auth比较特殊
|
||||
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
|
||||
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
if ok {
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.ReqHost
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
}
|
||||
this.writer.WriteHeader(http.StatusUnauthorized)
|
||||
this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
|
||||
return true
|
||||
}
|
||||
|
||||
// Basic Auth 姣旇緝鐗规畩
|
||||
if ref.AuthPolicy.Type == serverconfigs.HTTPAuthTypeBasicAuth {
|
||||
method, ok := ref.AuthPolicy.Method().(*serverconfigs.HTTPAuthBasicMethod)
|
||||
if ok {
|
||||
var headerValue = "Basic realm=\""
|
||||
if len(method.Realm) > 0 {
|
||||
headerValue += method.Realm
|
||||
} else {
|
||||
headerValue += this.ReqHost
|
||||
}
|
||||
headerValue += "\""
|
||||
if len(method.Charset) > 0 {
|
||||
headerValue += ", charset=\"" + method.Charset + "\""
|
||||
}
|
||||
this.writer.Header()["WWW-Authenticate"] = []string{headerValue}
|
||||
}
|
||||
}
|
||||
this.writer.WriteHeader(http.StatusUnauthorized)
|
||||
this.tags = append(this.tags, "auth:"+ref.AuthPolicy.Type)
|
||||
return true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/nodeconfigs"
|
||||
"github.com/TeaOSLab/EdgeCommon/pkg/rpc/pb"
|
||||
networksecurity "github.com/TeaOSLab/EdgeNode/internal/network-security"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/rpc"
|
||||
)
|
||||
|
||||
func (this *Node) execNetworkSecurityPolicyChangedTask(rpcClient *rpc.RPCClient) error {
|
||||
remotelogs.Println("NODE", "updating network security policy ...")
|
||||
resp, err := rpcClient.NodeRPC.FindNodeNetworkSecurityPolicy(rpcClient.Context(), &pb.FindNodeNetworkSecurityPolicyRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var policy = nodeconfigs.NewNetworkSecurityPolicy()
|
||||
if len(resp.NetworkSecurityPolicyJSON) > 0 {
|
||||
err = json.Unmarshal(resp.NetworkSecurityPolicyJSON, policy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
sharedNodeConfig.NetworkSecurityPolicy = policy
|
||||
|
||||
go networksecurity.SharedManager.Apply(policy)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && !packet
|
||||
//go:build plus
|
||||
|
||||
package nodes
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package netpackets
|
||||
|
||||
type FilterInterface interface {
|
||||
FilterMeta(meta *PacketMeta)
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus
|
||||
|
||||
package netpackets
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/linkedlist"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type IgnoredIPList struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
itemMap map[string]*linkedlist.Item[string] // linked => item
|
||||
list *linkedlist.List[string]
|
||||
|
||||
capacity int
|
||||
lastIP string
|
||||
}
|
||||
|
||||
func NewIgnoredIPList(capacity int) *IgnoredIPList {
|
||||
return &IgnoredIPList{
|
||||
itemMap: map[string]*linkedlist.Item[string]{},
|
||||
list: linkedlist.NewList[string](),
|
||||
capacity: capacity,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *IgnoredIPList) Add(ip string) {
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
if this.lastIP == ip {
|
||||
return
|
||||
}
|
||||
this.lastIP = ip
|
||||
|
||||
item, ok := this.itemMap[ip]
|
||||
if !ok {
|
||||
if this.capacity > 0 && len(this.itemMap) == this.capacity {
|
||||
var firstItem = this.list.Shift()
|
||||
if firstItem != nil {
|
||||
delete(this.itemMap, firstItem.Value)
|
||||
}
|
||||
}
|
||||
|
||||
item = linkedlist.NewItem[string](ip)
|
||||
this.itemMap[ip] = item
|
||||
}
|
||||
this.list.Push(item)
|
||||
}
|
||||
|
||||
func (this *IgnoredIPList) Remove(ip string) {
|
||||
this.mu.Lock()
|
||||
defer this.mu.Unlock()
|
||||
|
||||
item, ok := this.itemMap[ip]
|
||||
if ok {
|
||||
delete(this.itemMap, ip)
|
||||
this.list.Remove(item)
|
||||
}
|
||||
}
|
||||
|
||||
func (this *IgnoredIPList) Contains(ip string) bool {
|
||||
this.mu.RLock()
|
||||
defer this.mu.RUnlock()
|
||||
_, ok := this.itemMap[ip]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (this *IgnoredIPList) List(size int) (ipList []string) {
|
||||
if size <= 0 {
|
||||
return
|
||||
}
|
||||
|
||||
this.mu.RLock()
|
||||
defer this.mu.RUnlock()
|
||||
|
||||
this.list.RangeReverse(func(item *linkedlist.Item[string]) (goNext bool) {
|
||||
ipList = append(ipList, item.Value)
|
||||
size--
|
||||
return size > 0
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *IgnoredIPList) Len() int {
|
||||
this.mu.RLock()
|
||||
defer this.mu.RUnlock()
|
||||
return len(this.itemMap)
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus
|
||||
|
||||
package netpackets_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/netpackets"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"github.com/iwind/TeaGo/rands"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIgnoredIPList_Add(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
var list = netpackets.NewIgnoredIPList(10)
|
||||
list.Add("192.168.2.1")
|
||||
list.Add("192.168.2.2")
|
||||
list.Add("192.168.2.3")
|
||||
|
||||
a.IsTrue(list.Contains("192.168.2.1"))
|
||||
a.IsFalse(list.Contains("192.168.2.0"))
|
||||
|
||||
t.Log(list.List(0))
|
||||
t.Log(list.List(2))
|
||||
t.Log(list.List(4))
|
||||
}
|
||||
|
||||
func TestIgnoredIPList_Add_Capacity(t *testing.T) {
|
||||
var list = netpackets.NewIgnoredIPList(4)
|
||||
list.Add("192.168.2.1")
|
||||
list.Add("192.168.2.2")
|
||||
list.Add("192.168.2.3")
|
||||
list.Add("192.168.2.4")
|
||||
list.Add("192.168.2.5")
|
||||
list.Add("192.168.2.6")
|
||||
list.Add("192.168.2.7")
|
||||
t.Log(list.List(10))
|
||||
t.Log(list.Len(), "items")
|
||||
}
|
||||
|
||||
func TestIgnoredIPList_Remove(t *testing.T) {
|
||||
var list = netpackets.NewIgnoredIPList(10)
|
||||
list.Add("192.168.2.1")
|
||||
list.Add("192.168.2.2")
|
||||
list.Add("192.168.2.3")
|
||||
list.Remove("192.168.2.2")
|
||||
t.Log(list.List(4))
|
||||
}
|
||||
|
||||
func BenchmarkIgnoredIPList_Add(b *testing.B) {
|
||||
runtime.GOMAXPROCS(1)
|
||||
|
||||
var genIPFunc = func() string {
|
||||
return fmt.Sprintf("%d.%d.%d.%d", rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255), rands.Int(0, 255))
|
||||
}
|
||||
|
||||
var list = netpackets.NewIgnoredIPList(65535)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
list.Add(genIPFunc())
|
||||
list.Remove(genIPFunc())
|
||||
list.Contains(genIPFunc())
|
||||
if rands.Int(0, 100) == 0 {
|
||||
list.List(1000)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package netpackets
|
||||
|
||||
import (
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
)
|
||||
|
||||
type LayerType = gopacket.LayerType
|
||||
|
||||
var (
|
||||
LayerTypeTCP = layers.LayerTypeTCP
|
||||
LayerTypeUDP = layers.LayerTypeUDP
|
||||
LayerTypeICMPv4 = layers.LayerTypeICMPv4
|
||||
LayerTypeICMPv6 = layers.LayerTypeICMPv6
|
||||
)
|
||||
@@ -1,293 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package netpackets
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/remotelogs"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils"
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcap"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultBPFFilter = "(tcp or udp or icmp) and not net 127.0.0.1"
|
||||
|
||||
type Listener struct {
|
||||
filters []FilterInterface
|
||||
incomingHandle *pcap.Handle
|
||||
bpfFilter string
|
||||
|
||||
decodeDstIP bool
|
||||
|
||||
isClosed bool
|
||||
|
||||
outgoingIPList *IgnoredIPList
|
||||
outgoingHandle *pcap.Handle
|
||||
|
||||
filterTicker *time.Ticker
|
||||
}
|
||||
|
||||
func NewListener() *Listener {
|
||||
return &Listener{
|
||||
isClosed: true,
|
||||
outgoingIPList: NewIgnoredIPList(65535),
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Listener) Start() error {
|
||||
if !this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
this.isClosed = false
|
||||
|
||||
go func() {
|
||||
startErr := this.loopOutgoing()
|
||||
if startErr != nil {
|
||||
remotelogs.Error("NET_PACKET", "start outgoing packet listener failed: "+startErr.Error())
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
this.loopUpdateFilter()
|
||||
}()
|
||||
|
||||
for { // 无限 for 是为了防止意外退出
|
||||
err := this.loopIncoming()
|
||||
if err != nil {
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("start packet listener failed: %w", err)
|
||||
}
|
||||
|
||||
if this.isClosed {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Listener) AddFilter(filter FilterInterface) {
|
||||
this.filters = append(this.filters, filter)
|
||||
}
|
||||
|
||||
func (this *Listener) SetBPF(bpfFilter string) {
|
||||
this.bpfFilter = bpfFilter
|
||||
}
|
||||
|
||||
func (this *Listener) DecodeDstIP() {
|
||||
this.decodeDstIP = true
|
||||
}
|
||||
|
||||
func (this *Listener) Stop() {
|
||||
this.isClosed = true
|
||||
this.incomingHandle.Close()
|
||||
this.outgoingHandle.Close()
|
||||
}
|
||||
|
||||
func (this *Listener) IsRunning() bool {
|
||||
return this.incomingHandle != nil && !this.isClosed
|
||||
}
|
||||
|
||||
func (this *Listener) loopIncoming() error {
|
||||
const device = "any"
|
||||
var err error
|
||||
this.incomingHandle, err = pcap.OpenLive(device, 128, false /** ignore collision domain **/, pcap.BlockForever)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
this.incomingHandle.Close()
|
||||
}()
|
||||
|
||||
err = this.incomingHandle.SetDirection(pcap.DirectionIn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(this.bpfFilter) > 0 {
|
||||
err = this.incomingHandle.SetBPFFilter(this.bpfFilter)
|
||||
} else {
|
||||
this.bpfFilter = defaultBPFFilter
|
||||
err = this.incomingHandle.SetBPFFilter(defaultBPFFilter)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var meta = &PacketMeta{}
|
||||
|
||||
var packetSource = gopacket.NewPacketSource(this.incomingHandle, this.incomingHandle.LinkType())
|
||||
packetSource.NoCopy = true
|
||||
packetSource.Lazy = true
|
||||
|
||||
var filters = this.filters
|
||||
var countFilters = len(filters)
|
||||
|
||||
for packet := range packetSource.Packets() {
|
||||
var networkLayer = packet.NetworkLayer()
|
||||
if networkLayer == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var networkFlow = networkLayer.NetworkFlow()
|
||||
|
||||
var src = networkFlow.Src()
|
||||
|
||||
meta.SrcIP = src.String()
|
||||
|
||||
// ignore outgoing ip
|
||||
if this.outgoingIPList.Contains(meta.SrcIP) {
|
||||
continue
|
||||
}
|
||||
|
||||
if this.decodeDstIP {
|
||||
meta.DstIP = networkFlow.Dst().String()
|
||||
}
|
||||
meta.Length = packet.Metadata().Length
|
||||
|
||||
var transportLayer = packet.TransportLayer()
|
||||
if transportLayer == nil {
|
||||
meta.SrcPort = 0
|
||||
meta.DstPort = 0
|
||||
|
||||
switch x := networkLayer.(type) {
|
||||
case *layers.IPv4:
|
||||
meta.LayerType = x.NextLayerType()
|
||||
case *layers.IPv6:
|
||||
meta.LayerType = x.NextLayerType()
|
||||
}
|
||||
|
||||
// call filters
|
||||
if countFilters == 1 {
|
||||
filters[0].FilterMeta(meta)
|
||||
} else {
|
||||
for _, filter := range filters {
|
||||
filter.FilterMeta(meta)
|
||||
}
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var transportFlow = transportLayer.TransportFlow()
|
||||
meta.SrcPort = int(binary.BigEndian.Uint16(transportFlow.Src().Raw()))
|
||||
meta.LayerType = transportLayer.LayerType()
|
||||
meta.DstPort = int(binary.BigEndian.Uint16(transportFlow.Dst().Raw()))
|
||||
|
||||
// call filters
|
||||
if countFilters == 1 {
|
||||
filters[0].FilterMeta(meta)
|
||||
} else {
|
||||
for _, filter := range filters {
|
||||
filter.FilterMeta(meta)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Listener) loopOutgoing() error {
|
||||
const device = "any"
|
||||
var err error
|
||||
this.outgoingHandle, err = pcap.OpenLive(device, 128, true /** ignore collision domain **/, pcap.BlockForever)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
this.outgoingHandle.Close()
|
||||
}()
|
||||
|
||||
err = this.outgoingHandle.SetDirection(pcap.DirectionOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = this.outgoingHandle.SetBPFFilter("tcp and tcp[tcpflags] & (tcp-syn) != 0 and tcp[tcpflags] & (tcp-ack) = 0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var packetSource = gopacket.NewPacketSource(this.outgoingHandle, this.outgoingHandle.LinkType())
|
||||
packetSource.NoCopy = true
|
||||
packetSource.Lazy = true
|
||||
|
||||
for packet := range packetSource.Packets() {
|
||||
var networkLayer = packet.NetworkLayer()
|
||||
if networkLayer == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var networkFlow = networkLayer.NetworkFlow()
|
||||
var dstIP = networkFlow.Dst().String()
|
||||
this.outgoingIPList.Add(dstIP)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Listener) loopUpdateFilter() {
|
||||
if this.filterTicker != nil {
|
||||
return
|
||||
}
|
||||
|
||||
this.filterTicker = time.NewTicker(1 * time.Second)
|
||||
var lastIPList []string
|
||||
for range this.filterTicker.C {
|
||||
if this.isClosed {
|
||||
continue
|
||||
}
|
||||
|
||||
var ipList = this.outgoingIPList.List(512) // 基于bfp长度的限制,这里数量不能太多
|
||||
sort.Strings(ipList)
|
||||
if this.equalStrings(lastIPList, ipList) {
|
||||
continue
|
||||
}
|
||||
|
||||
lastIPList = ipList
|
||||
|
||||
// apply
|
||||
var incomingHandle = this.incomingHandle
|
||||
if incomingHandle != nil {
|
||||
var rules = []string{}
|
||||
for _, ip := range ipList {
|
||||
rules = append(rules, "not src host "+ip)
|
||||
}
|
||||
var newBPFFilter = this.bpfFilter + " and " + strings.Join(rules, " and ")
|
||||
if utils.IsDebugEnv() {
|
||||
remotelogs.Debug("NET_PACKET", "set new BPF filter: "+newBPFFilter)
|
||||
}
|
||||
err := incomingHandle.SetBPFFilter(newBPFFilter)
|
||||
if err != nil {
|
||||
remotelogs.Error("NET_PACKET", "set new BPF filter failed: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (this *Listener) equalStrings(s1 []string, s2 []string) bool {
|
||||
var l = len(s1)
|
||||
if len(s2) != l {
|
||||
return false
|
||||
}
|
||||
if l == 0 {
|
||||
return true
|
||||
}
|
||||
for i := 0; i < l; i++ {
|
||||
if s1[i] != s2[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package netpackets_test
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/netpackets"
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/testutils"
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/iwind/TeaGo/types"
|
||||
"log"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type testFilter struct {
|
||||
}
|
||||
|
||||
func (this *testFilter) FilterMeta(meta *netpackets.PacketMeta) {
|
||||
log.Println(meta.LayerType.String() + " " + meta.SrcIP + ":" + types.String(meta.SrcPort) + " => " + meta.DstIP + ":" + types.String(meta.DstPort) + " " + types.String(meta.Length) + "bytes")
|
||||
}
|
||||
|
||||
func TestListener_Start(t *testing.T) {
|
||||
if !testutils.IsSingleTesting() {
|
||||
if os.Getgid() > 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var listener = netpackets.NewListener()
|
||||
listener.AddFilter(&testFilter{})
|
||||
|
||||
go func() {
|
||||
time.Sleep(10 * time.Second)
|
||||
t.Log("stopping ...")
|
||||
listener.Stop()
|
||||
}()
|
||||
|
||||
t.Log("starting ...")
|
||||
err := listener.Start()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListener_DecodePacket_UDP(t *testing.T) {
|
||||
var packetData = []byte{69, 0, 0, 134, 140, 133, 0, 0, 118, 17, 16, 79, 223, 5, 5, 5, 192, 168, 2, 224, 0, 53, 232, 163, 0, 114, 0, 0, 69, 42, 129, 128, 0, 1, 0, 3, 0, 0, 0, 0, 6, 115, 116, 97, 116, 105, 99, 7, 111, 115, 99, 104, 105, 110, 97, 3, 110, 101, 116, 0, 0, 1, 0, 1, 192, 12, 0, 5, 0, 1, 0, 0, 0, 1, 0, 25, 10, 115, 116, 97, 116, 105, 99, 45, 111, 115, 99, 2, 98, 48, 5, 97, 105, 99, 100, 110, 3, 99, 111, 109, 0, 192, 48, 0, 5, 0, 1, 0, 0, 0, 1, 0, 5, 2, 118, 109, 192, 62, 192, 85, 0, 1, 0, 1, 0, 0, 0, 1, 0, 4, 218, 28, 104, 157}
|
||||
|
||||
var packet = gopacket.NewPacket(packetData, layers.LayerTypeIPv4, gopacket.DecodeOptions{})
|
||||
var networkFlow = packet.NetworkLayer().NetworkFlow()
|
||||
|
||||
t.Log(networkFlow)
|
||||
|
||||
t.Log(packet.Metadata().Length)
|
||||
t.Log(packet.TransportLayer().TransportFlow())
|
||||
}
|
||||
|
||||
func TestListener_DecodePacket_TCP(t *testing.T) {
|
||||
var packetData = []byte{69, 8, 0, 64, 6, 51, 64, 0, 52, 6, 188, 222, 74, 91, 117, 187, 192, 168, 2, 224, 1, 187, 225, 226, 137, 198, 251, 25, 221, 137, 133, 93, 176, 16, 1, 245, 224, 6, 0, 0, 1, 1, 8, 10, 30, 187, 162, 175, 35, 215, 100, 174, 1, 1, 5, 10, 221, 137, 133, 68, 221, 137, 133, 93}
|
||||
|
||||
var packet = gopacket.NewPacket(packetData, layers.LayerTypeIPv4, gopacket.DecodeOptions{})
|
||||
var networkFlow = packet.NetworkLayer().NetworkFlow()
|
||||
t.Log(networkFlow.Src().Raw(), len(networkFlow.Src().Raw()))
|
||||
|
||||
t.Log(networkFlow)
|
||||
|
||||
t.Log(packet.Metadata().Length)
|
||||
t.Log(packet.TransportLayer().TransportFlow())
|
||||
}
|
||||
|
||||
func BenchmarkListener_DecodePacket(b *testing.B) {
|
||||
var packetData = []byte{69, 0, 0, 134, 140, 133, 0, 0, 118, 17, 16, 79, 223, 5, 5, 5, 192, 168, 2, 224, 0, 53, 232, 163, 0, 114, 0, 0, 69, 42, 129, 128, 0, 1, 0, 3, 0, 0, 0, 0, 6, 115, 116, 97, 116, 105, 99, 7, 111, 115, 99, 104, 105, 110, 97, 3, 110, 101, 116, 0, 0, 1, 0, 1, 192, 12, 0, 5, 0, 1, 0, 0, 0, 1, 0, 25, 10, 115, 116, 97, 116, 105, 99, 45, 111, 115, 99, 2, 98, 48, 5, 97, 105, 99, 100, 110, 3, 99, 111, 109, 0, 192, 48, 0, 5, 0, 1, 0, 0, 0, 1, 0, 5, 2, 118, 109, 192, 62, 192, 85, 0, 1, 0, 1, 0, 0, 0, 1, 0, 4, 218, 28, 104, 157}
|
||||
|
||||
var decodeOptions = gopacket.DecodeOptions{
|
||||
Lazy: true,
|
||||
NoCopy: true,
|
||||
//SkipDecodeRecovery: true,
|
||||
}
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
var packet = gopacket.NewPacket(packetData, layers.LayerTypeIPv4, decodeOptions)
|
||||
|
||||
var networkFlow = packet.NetworkLayer().NetworkFlow()
|
||||
var src = networkFlow.Src()
|
||||
var dest = networkFlow.Dst()
|
||||
|
||||
_ = netpackets.IsLocalRawIPv4(src.Raw())
|
||||
_ = netpackets.IsLocalRawIPv4(dest.Raw())
|
||||
|
||||
_ = src.String()
|
||||
_ = dest.String()
|
||||
|
||||
_ = packet.Metadata().Length
|
||||
|
||||
var transportFlow = packet.TransportLayer().TransportFlow()
|
||||
//_ = transportFlow.Src().String()
|
||||
//_ = transportFlow.Dst().String()
|
||||
|
||||
_ = int(binary.BigEndian.Uint16(transportFlow.Src().Raw()))
|
||||
_ = int(binary.BigEndian.Uint16(transportFlow.Dst().Raw()))
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
//go:build plus && packet
|
||||
|
||||
package netpackets
|
||||
|
||||
type PacketMeta struct {
|
||||
LayerType LayerType
|
||||
SrcIP string
|
||||
SrcPort int
|
||||
DstIP string
|
||||
DstPort int
|
||||
Length int
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package netpackets
|
||||
|
||||
// IsLocalRawIPv4 使用原始IP数据判断是否为本地IPv4
|
||||
func IsLocalRawIPv4(ip []byte) bool {
|
||||
if len(ip) != 4 {
|
||||
return false
|
||||
}
|
||||
if ip[0] == 127 ||
|
||||
ip[0] == 10 ||
|
||||
(ip[0] == 172 && ip[1]&0xf0 == 16) ||
|
||||
(ip[0] == 192 && ip[1] == 168) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
// Copyright 2023 GoEdge CDN goedge.cdn@gmail.com. All rights reserved. Official site: https://goedge.cn .
|
||||
|
||||
package netpackets_test
|
||||
|
||||
import (
|
||||
"github.com/TeaOSLab/EdgeNode/internal/utils/netpackets"
|
||||
"github.com/iwind/TeaGo/assert"
|
||||
"net"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsLocalRawIPv4(t *testing.T) {
|
||||
var a = assert.NewAssertion(t)
|
||||
|
||||
a.IsTrue(netpackets.IsLocalRawIPv4(net.ParseIP("192.168.2.100").To4()))
|
||||
a.IsTrue(netpackets.IsLocalRawIPv4(net.ParseIP("127.0.0.1").To4()))
|
||||
a.IsTrue(netpackets.IsLocalRawIPv4(net.ParseIP("172.16.0.1").To4()))
|
||||
a.IsTrue(netpackets.IsLocalRawIPv4(net.ParseIP("10.0.0.1").To4()))
|
||||
|
||||
a.IsFalse(netpackets.IsLocalRawIPv4(net.ParseIP("1.2.3.4").To4()))
|
||||
}
|
||||
Reference in New Issue
Block a user