From c68dbc75c3b5ab2266dc950520622f77d7f632ce Mon Sep 17 00:00:00 2001 From: Nene7ko_ <1604009816@qq.com> Date: Mon, 10 Aug 2026 21:19:45 +0800 Subject: [PATCH 1/2] test(stack): secure Router Nacos transports --- .env.example | 1 + .github/workflows/ci.yml | 10 +- README.md | 25 +- cmd/nacos-secure-fixture/main.go | 288 ++++++++++++++++++ cmd/nacos-secure-fixture/main_test.go | 206 +++++++++++++ compose.yaml | 46 ++- scripts/prepare.sh | 3 + scripts/write-ci-env.sh | 5 + tests/backend/invoke_record_test.go | 61 ++-- tests/backend/nacos_tls_fixture_test.go | 19 +- .../fixtures/nacos-secure-fixture.Dockerfile | 15 + 11 files changed, 646 insertions(+), 33 deletions(-) create mode 100644 cmd/nacos-secure-fixture/main.go create mode 100644 cmd/nacos-secure-fixture/main_test.go create mode 100644 tests/fixtures/nacos-secure-fixture.Dockerfile diff --git a/.env.example b/.env.example index 878074a..5caf2d7 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,7 @@ NEKIRO_CONTROL_PLANE_IMAGE=replace-with-prepared-image NEKIRO_A2A_ROUTER_IMAGE=replace-with-prepared-image NEKIRO_RUNTIME_A_IMAGE=replace-with-prepared-image NEKIRO_RUNTIME_B_IMAGE=replace-with-prepared-image +NEKIRO_NACOS_SECURE_PROXY_IMAGE=replace-with-prepared-image # Select exactly one Router instance-routing mode. Use direct to dispatch to # the exact published Card endpoint, or config_center_file to select an diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2800a4..bd278d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,11 +116,14 @@ jobs: run: | ./scripts/prepare.sh "$GITHUB_WORKSPACE/components.json" "$WORK_ROOT" "$PREPARED_ENV" source "$PREPARED_ENV" - for name in NEKIRO_CONTROL_PLANE_IMAGE NEKIRO_A2A_ROUTER_IMAGE NEKIRO_RUNTIME_A_IMAGE NEKIRO_RUNTIME_B_IMAGE NEKIRO_STACK_CORE_DIR NEKIRO_STACK_CONSOLE_DIR NEKIRO_STACK_SDK_GO_DIR NEKIRO_STACK_SAMPLES_DIR NEKIRO_STACK_TRANSPORT_GO_DIR; do + for name in NEKIRO_CONTROL_PLANE_IMAGE NEKIRO_A2A_ROUTER_IMAGE NEKIRO_RUNTIME_A_IMAGE NEKIRO_RUNTIME_B_IMAGE NEKIRO_NACOS_SECURE_PROXY_IMAGE NEKIRO_STACK_CORE_DIR NEKIRO_STACK_CONSOLE_DIR NEKIRO_STACK_SDK_GO_DIR NEKIRO_STACK_SAMPLES_DIR NEKIRO_STACK_TRANSPORT_GO_DIR; do printf '%s=%s\n' "$name" "${!name}" >>"$GITHUB_ENV" done printf 'NEKIRO_STACK_PREPARED_ENV=%s\n' "$PREPARED_ENV" >>"$GITHUB_ENV" + - name: Generate ephemeral Nacos PKI + run: go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" + - name: Start fresh exact backend assembly run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml up --detach --wait --wait-timeout 120 @@ -173,11 +176,14 @@ jobs: run: | ./scripts/prepare.sh "$GITHUB_WORKSPACE/components.json" "$WORK_ROOT" "$PREPARED_ENV" source "$PREPARED_ENV" - for name in NEKIRO_CONTROL_PLANE_IMAGE NEKIRO_A2A_ROUTER_IMAGE NEKIRO_RUNTIME_A_IMAGE NEKIRO_RUNTIME_B_IMAGE NEKIRO_STACK_CORE_DIR NEKIRO_STACK_CONSOLE_DIR NEKIRO_STACK_SDK_GO_DIR NEKIRO_STACK_SAMPLES_DIR NEKIRO_STACK_TRANSPORT_GO_DIR; do + for name in NEKIRO_CONTROL_PLANE_IMAGE NEKIRO_A2A_ROUTER_IMAGE NEKIRO_RUNTIME_A_IMAGE NEKIRO_RUNTIME_B_IMAGE NEKIRO_NACOS_SECURE_PROXY_IMAGE NEKIRO_STACK_CORE_DIR NEKIRO_STACK_CONSOLE_DIR NEKIRO_STACK_SDK_GO_DIR NEKIRO_STACK_SAMPLES_DIR NEKIRO_STACK_TRANSPORT_GO_DIR; do printf '%s=%s\n' "$name" "${!name}" >>"$GITHUB_ENV" done printf 'NEKIRO_STACK_PREPARED_ENV=%s\n' "$PREPARED_ENV" >>"$GITHUB_ENV" + - name: Generate ephemeral Nacos PKI + run: go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" + - name: Start fresh exact browser assembly run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml up --detach --wait --wait-timeout 120 diff --git a/README.md b/README.md index ac50c5d..a569526 100644 --- a/README.md +++ b/README.md @@ -60,15 +60,15 @@ docker compose --project-name nekiro-stack --file compose.yaml up --detach --wai go test -tags=e2e -count=1 ./tests/backend ``` -`NEKIRO_ROUTER_INSTANCE_ROUTING_MODE` must be either `direct` or -`config_center_file`; there is no automatic fallback between them. File mode +`NEKIRO_ROUTER_INSTANCE_ROUTING_MODE` must be `direct`, `config_center_file`, or +`nacos`; there is no automatic fallback between them. File mode reads the exact Release-scoped `router-instance-directory.v1` document from the absolute host directory named by `NEKIRO_ROUTER_CONFIG_CENTER_ROOT`. Missing, malformed, empty, or ambiguous instance data makes new Invocations -fail closed. The backend acceptance scenario publishes one ready Runtime B -replica into that directory and verifies both JSON and SSE dispatch to its -reported instance ID. Compose and browser scenarios select `direct` -explicitly because instance-directory behavior is outside those checks. +fail closed. The backend scenario selects `nacos` and uses the Stack-owned +secure Nacos fixture for Router Config Center reads, initial Naming snapshots, +and gRPC watch updates. Browser scenarios select `direct` because instance +directory behavior is outside those checks. ## Test matrix and success signals @@ -88,11 +88,22 @@ assembly. Backend runtime registration uses an ephemeral private PKI generated by the acceptance process. Runtime A publishes through an authenticated TLS Nacos endpoint; Runtime B and its replacement publish through an mTLS endpoint. The -Router observes those leases before managed invocation. Negative containers +Router reaches Config Center and Naming through authenticated HTTP and gRPC +mTLS endpoints before observing those leases. Negative containers with a wrong CA, wrong server name, or missing mTLS client identity must fail before publication and leave no routable instance. Test keys exist only in the caller-owned runtime directory and are never committed or recorded in Ledger. +For a local backend run, generate the ephemeral material before Compose starts: + +```bash +go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" +``` + +The secure fixture exposes only a status endpoint on the host. Its counters +prove that the Router completed authenticated HTTP reads and established the +authenticated gRPC watch; Nacos itself remains internal to Compose. + The `Core integration` reusable workflow accepts a full Core commit SHA, renders a temporary immutable manifest, and runs both backend and browser acceptance. Core calls it after every merge to `main`; the tracked Stack diff --git a/cmd/nacos-secure-fixture/main.go b/cmd/nacos-secure-fixture/main.go new file mode 100644 index 0000000..12cd1c0 --- /dev/null +++ b/cmd/nacos-secure-fixture/main.go @@ -0,0 +1,288 @@ +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "log" + "math/big" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/signal" + "path/filepath" + "sync/atomic" + "syscall" + "time" +) + +const maxTLSFileBytes = 1 << 20 + +type counters struct { + httpRequests atomic.Int64 + grpcConnections atomic.Int64 +} + +func main() { + if len(os.Args) < 2 { + log.Fatal("usage: nacos-secure-fixture ") + } + var err error + switch os.Args[1] { + case "generate": + if len(os.Args) != 3 { + log.Fatal("usage: nacos-secure-fixture generate ") + } + err = generateMaterial(os.Args[2], time.Now()) + case "serve": + err = serve() + default: + log.Fatal("unsupported nacos-secure-fixture command") + } + if err != nil { + log.Fatal(err) + } +} + +func generateMaterial(directory string, now time.Time) error { + if !filepath.IsAbs(directory) || filepath.Clean(directory) != directory { + return errors.New("output directory must be a clean absolute path") + } + // The directory is mounted read-only into non-root fixture containers. The + // material is ephemeral and removed with the CI workspace. + if err := os.MkdirAll(directory, 0o755); err != nil { + return errors.New("create TLS output directory") + } + caPublic, caPrivate, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return errors.New("generate CA key") + } + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "NeKiro Stack Nacos E2E CA"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(24 * time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, caPublic, caPrivate) + if err != nil { + return errors.New("create CA certificate") + } + caCertificate, err := x509.ParseCertificate(caDER) + if err != nil { + return errors.New("parse CA certificate") + } + if err := writeExclusive(filepath.Join(directory, "ca.pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o444); err != nil { + return err + } + if err := issueCertificate(directory, "server", 2, x509.ExtKeyUsageServerAuth, []string{"nacos.internal"}, caCertificate, caPrivate, now); err != nil { + return err + } + if err := issueCertificate(directory, "client", 3, x509.ExtKeyUsageClientAuth, nil, caCertificate, caPrivate, now); err != nil { + return err + } + otherPublic, otherPrivate, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return errors.New("generate negative CA key") + } + otherTemplate := &x509.Certificate{SerialNumber: big.NewInt(4), Subject: pkix.Name{CommonName: "Wrong CA"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(24 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + otherDER, err := x509.CreateCertificate(rand.Reader, otherTemplate, otherTemplate, otherPublic, otherPrivate) + if err != nil { + return errors.New("create negative CA certificate") + } + return writeExclusive(filepath.Join(directory, "wrong-ca.pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: otherDER}), 0o444) +} + +func issueCertificate(directory, name string, serial int64, usage x509.ExtKeyUsage, dnsNames []string, ca *x509.Certificate, caPrivate ed25519.PrivateKey, now time.Time) error { + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return errors.New("generate leaf key") + } + template := &x509.Certificate{SerialNumber: big.NewInt(serial), Subject: pkix.Name{CommonName: name}, DNSNames: dnsNames, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{usage}} + der, err := x509.CreateCertificate(rand.Reader, template, ca, public, caPrivate) + if err != nil { + return errors.New("create leaf certificate") + } + keyDER, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + return errors.New("marshal leaf key") + } + if err := writeExclusive(filepath.Join(directory, name+".pem"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o444); err != nil { + return err + } + return writeExclusive(filepath.Join(directory, name+"-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o444) +} + +func writeExclusive(path string, content []byte, mode os.FileMode) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return errors.New("create TLS material") + } + if _, err := file.Write(content); err != nil { + _ = file.Close() + return errors.New("write TLS material") + } + if err := file.Close(); err != nil { + return errors.New("close TLS material") + } + return nil +} + +func serve() error { + tlsRoot := os.Getenv("NEKIRO_NACOS_FIXTURE_TLS_ROOT") + if !filepath.IsAbs(tlsRoot) || filepath.Clean(tlsRoot) != tlsRoot { + return errors.New("TLS root must be a clean absolute path") + } + serverCertificate, caPool, err := loadServerTLS(tlsRoot) + if err != nil { + return err + } + sharedTLS := &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{serverCertificate}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: caPool} + grpcTLS := sharedTLS.Clone() + grpcTLS.NextProtos = []string{"h2"} + metrics := &counters{} + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + errCh := make(chan error, 3) + go func() { + errCh <- serveHTTPProxy(ctx, requiredEnv("NEKIRO_NACOS_FIXTURE_HTTP_LISTEN"), requiredEnv("NEKIRO_NACOS_FIXTURE_HTTP_UPSTREAM"), sharedTLS.Clone(), metrics) + }() + go func() { + errCh <- serveGRPCProxy(ctx, requiredEnv("NEKIRO_NACOS_FIXTURE_GRPC_LISTEN"), requiredEnv("NEKIRO_NACOS_FIXTURE_GRPC_UPSTREAM"), grpcTLS, metrics) + }() + go func() { errCh <- serveStatus(ctx, requiredEnv("NEKIRO_NACOS_FIXTURE_STATUS_LISTEN"), metrics) }() + select { + case <-ctx.Done(): + return nil + case err := <-errCh: + return err + } +} + +func loadServerTLS(root string) (tls.Certificate, *x509.CertPool, error) { + certificate, err := tls.LoadX509KeyPair(filepath.Join(root, "server.pem"), filepath.Join(root, "server-key.pem")) + if err != nil { + return tls.Certificate{}, nil, errors.New("load fixture server identity") + } + caPEM, err := readLimited(filepath.Join(root, "ca.pem")) + if err != nil { + return tls.Certificate{}, nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + return tls.Certificate{}, nil, errors.New("parse fixture private CA") + } + return certificate, pool, nil +} + +func readLimited(path string) ([]byte, error) { + file, err := os.Open(path) + if err != nil { + return nil, errors.New("open fixture TLS file") + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, maxTLSFileBytes+1)) + if err != nil || len(data) == 0 || len(data) > maxTLSFileBytes { + return nil, errors.New("read fixture TLS file") + } + return data, nil +} + +func serveHTTPProxy(ctx context.Context, listenAddress, upstream string, tlsConfig *tls.Config, metrics *counters) error { + target, err := url.Parse(upstream) + if err != nil || target.Scheme != "http" || target.Host == "" { + return errors.New("invalid HTTP upstream") + } + proxy := httputil.NewSingleHostReverseProxy(target) + proxy.Transport = &http.Transport{Proxy: nil} + proxy.ErrorHandler = func(writer http.ResponseWriter, _ *http.Request, _ error) { + http.Error(writer, "Nacos upstream unavailable", http.StatusBadGateway) + } + handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + metrics.httpRequests.Add(1) + proxy.ServeHTTP(writer, request) + }) + listener, err := net.Listen("tcp", listenAddress) + if err != nil { + return errors.New("listen on secure Nacos HTTP boundary") + } + server := &http.Server{Handler: handler} + go func() { <-ctx.Done(); _ = server.Close() }() + err = server.Serve(tls.NewListener(listener, tlsConfig)) + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func serveGRPCProxy(ctx context.Context, listenAddress, upstream string, tlsConfig *tls.Config, metrics *counters) error { + listener, err := net.Listen("tcp", listenAddress) + if err != nil { + return errors.New("listen on secure Nacos gRPC boundary") + } + go func() { <-ctx.Done(); _ = listener.Close() }() + for { + connection, err := listener.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return errors.New("accept secure Nacos gRPC connection") + } + go proxyGRPC(connection, upstream, tlsConfig, metrics) + } +} + +func proxyGRPC(connection net.Conn, upstream string, tlsConfig *tls.Config, metrics *counters) { + defer connection.Close() + secure := tls.Server(connection, tlsConfig) + if err := secure.Handshake(); err != nil { + return + } + backend, err := net.DialTimeout("tcp", upstream, 5*time.Second) + if err != nil { + return + } + defer backend.Close() + metrics.grpcConnections.Add(1) + done := make(chan struct{}, 1) + go func() { _, _ = io.Copy(backend, secure); _ = backend.(*net.TCPConn).CloseWrite(); done <- struct{}{} }() + _, _ = io.Copy(secure, backend) + <-done +} + +func serveStatus(ctx context.Context, listenAddress string, metrics *counters) error { + mux := http.NewServeMux() + mux.HandleFunc("/readyz", func(writer http.ResponseWriter, _ *http.Request) { writer.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("/status", func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(writer).Encode(map[string]int64{"httpRequests": metrics.httpRequests.Load(), "grpcConnections": metrics.grpcConnections.Load()}) + }) + server := &http.Server{Addr: listenAddress, Handler: mux} + go func() { <-ctx.Done(); _ = server.Close() }() + err := server.ListenAndServe() + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err +} + +func requiredEnv(name string) string { + value := os.Getenv(name) + if value == "" { + panic(fmt.Sprintf("%s must be set", name)) + } + return value +} diff --git a/cmd/nacos-secure-fixture/main_test.go b/cmd/nacos-secure-fixture/main_test.go new file mode 100644 index 0000000..a3b96bf --- /dev/null +++ b/cmd/nacos-secure-fixture/main_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +func TestGenerateMaterialProducesSeparateServerAndClientIdentities(t *testing.T) { + directory := filepath.Join(t.TempDir(), "tls") + now := time.Now().UTC().Truncate(time.Second) + if err := generateMaterial(directory, now); err != nil { + t.Fatal(err) + } + for _, name := range []string{"ca.pem", "server.pem", "server-key.pem", "client.pem", "client-key.pem", "wrong-ca.pem"} { + info, err := os.Stat(filepath.Join(directory, name)) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + t.Fatalf("generated %s info=%v err=%v", name, info, err) + } + } + server, err := tls.LoadX509KeyPair(filepath.Join(directory, "server.pem"), filepath.Join(directory, "server-key.pem")) + if err != nil { + t.Fatal(err) + } + serverLeaf, err := x509.ParseCertificate(server.Certificate[0]) + if err != nil { + t.Fatal(err) + } + if err := serverLeaf.VerifyHostname("nacos.internal"); err != nil { + t.Fatal(err) + } + client, err := tls.LoadX509KeyPair(filepath.Join(directory, "client.pem"), filepath.Join(directory, "client-key.pem")) + if err != nil { + t.Fatal(err) + } + clientLeaf, err := x509.ParseCertificate(client.Certificate[0]) + if err != nil { + t.Fatal(err) + } + if len(clientLeaf.ExtKeyUsage) != 1 || clientLeaf.ExtKeyUsage[0] != x509.ExtKeyUsageClientAuth { + t.Fatalf("client usages=%v", clientLeaf.ExtKeyUsage) + } +} + +func TestGenerateMaterialRejectsReuse(t *testing.T) { + directory := filepath.Join(t.TempDir(), "tls") + if err := generateMaterial(directory, time.Now()); err != nil { + t.Fatal(err) + } + if err := generateMaterial(directory, time.Now()); err == nil { + t.Fatal("second generation unexpectedly replaced existing credentials") + } +} + +func TestSecureHTTPProxyRequiresClientIdentityAndForwards(t *testing.T) { + directory := generateTestMaterial(t) + serverIdentity, pool, err := loadServerTLS(directory) + if err != nil { + t.Fatal(err) + } + upstream := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/nacos/ready" { + t.Fatalf("upstream path=%q", request.URL.Path) + } + writer.WriteHeader(http.StatusNoContent) + })) + defer upstream.Close() + address := unusedAddress(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + metrics := &counters{} + errCh := make(chan error, 1) + go func() { + errCh <- serveHTTPProxy(ctx, address, upstream.URL, &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{serverIdentity}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool}, metrics) + }() + client := &http.Client{Transport: &http.Transport{TLSClientConfig: clientTLS(t, directory, nil)}, Timeout: 5 * time.Second} + response, err := waitHTTP(client, "https://"+address+"/nacos/ready") + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusNoContent || metrics.httpRequests.Load() != 1 { + t.Fatalf("status=%d requests=%d", response.StatusCode, metrics.httpRequests.Load()) + } + withoutIdentity := &http.Client{Transport: &http.Transport{TLSClientConfig: clientTLS(t, directory, []tls.Certificate{})}, Timeout: time.Second} + if _, err := withoutIdentity.Get("https://" + address + "/nacos/ready"); err == nil { + t.Fatal("HTTP mTLS boundary accepted a client without an identity") + } +} + +func TestSecureGRPCProxyTerminatesMTLSAndForwardsBytes(t *testing.T) { + directory := generateTestMaterial(t) + backend, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer backend.Close() + go func() { + connection, acceptErr := backend.Accept() + if acceptErr != nil { + return + } + defer connection.Close() + _, _ = io.Copy(connection, connection) + }() + serverIdentity, pool, err := loadServerTLS(directory) + if err != nil { + t.Fatal(err) + } + address := unusedAddress(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + metrics := &counters{} + errCh := make(chan error, 1) + go func() { + errCh <- serveGRPCProxy(ctx, address, backend.Addr().String(), &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{serverIdentity}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool, NextProtos: []string{"h2"}}, metrics) + }() + var connection *tls.Conn + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + connection, err = tls.Dial("tcp", address, clientTLS(t, directory, nil)) + if err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + defer connection.Close() + if connection.ConnectionState().NegotiatedProtocol != "h2" { + t.Fatalf("negotiated protocol=%q", connection.ConnectionState().NegotiatedProtocol) + } + if _, err := connection.Write([]byte("grpc-fixture")); err != nil { + t.Fatal(err) + } + buffer := make([]byte, len("grpc-fixture")) + if _, err := io.ReadFull(connection, buffer); err != nil || string(buffer) != "grpc-fixture" { + t.Fatalf("forwarded=%q err=%v", buffer, err) + } + if metrics.grpcConnections.Load() != 1 { + t.Fatalf("gRPC connections=%d", metrics.grpcConnections.Load()) + } +} + +func generateTestMaterial(t *testing.T) string { + t.Helper() + directory := filepath.Join(t.TempDir(), "tls") + if err := generateMaterial(directory, time.Now()); err != nil { + t.Fatal(err) + } + return directory +} + +func clientTLS(t *testing.T, directory string, certificates []tls.Certificate) *tls.Config { + t.Helper() + caPEM, err := os.ReadFile(filepath.Join(directory, "ca.pem")) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + t.Fatal("append private CA") + } + if certificates == nil { + identity, err := tls.LoadX509KeyPair(filepath.Join(directory, "client.pem"), filepath.Join(directory, "client-key.pem")) + if err != nil { + t.Fatal(err) + } + certificates = []tls.Certificate{identity} + } + return &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: pool, ServerName: "nacos.internal", Certificates: certificates, NextProtos: []string{"h2"}} +} + +func unusedAddress(t *testing.T) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + address := listener.Addr().String() + _ = listener.Close() + return address +} + +func waitHTTP(client *http.Client, endpoint string) (*http.Response, error) { + deadline := time.Now().Add(5 * time.Second) + var err error + for time.Now().Before(deadline) { + var response *http.Response + response, err = client.Get(endpoint) + if err == nil { + return response, nil + } + time.Sleep(20 * time.Millisecond) + } + return nil, err +} diff --git a/compose.yaml b/compose.yaml index 73da106..28213a4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -40,6 +40,35 @@ services: - platform-internal restart: "no" + nacos-secure: + image: "${NEKIRO_NACOS_SECURE_PROXY_IMAGE:?NEKIRO_NACOS_SECURE_PROXY_IMAGE must be set to the prepared immutable image}" + command: ["serve"] + environment: + NEKIRO_NACOS_FIXTURE_TLS_ROOT: "/var/run/nekiro-nacos-tls" + NEKIRO_NACOS_FIXTURE_HTTP_LISTEN: "0.0.0.0:9443" + NEKIRO_NACOS_FIXTURE_HTTP_UPSTREAM: "http://nacos:8848" + NEKIRO_NACOS_FIXTURE_GRPC_LISTEN: "0.0.0.0:9848" + NEKIRO_NACOS_FIXTURE_GRPC_UPSTREAM: "nacos:9848" + NEKIRO_NACOS_FIXTURE_STATUS_LISTEN: "0.0.0.0:9090" + volumes: + - "${NEKIRO_E2E_TLS_ROOT:?NEKIRO_E2E_TLS_ROOT must be set and non-empty}:/var/run/nekiro-nacos-tls:ro" + ports: + - "127.0.0.1:${NEKIRO_E2E_NACOS_ROUTER_HTTP_PORT:?NEKIRO_E2E_NACOS_ROUTER_HTTP_PORT must be set and non-empty}:9443" + - "127.0.0.1:${NEKIRO_E2E_NACOS_ROUTER_GRPC_PORT:?NEKIRO_E2E_NACOS_ROUTER_GRPC_PORT must be set and non-empty}:9848" + - "127.0.0.1:${NEKIRO_E2E_NACOS_FIXTURE_STATUS_PORT:?NEKIRO_E2E_NACOS_FIXTURE_STATUS_PORT must be set and non-empty}:9090" + depends_on: + nacos: + condition: service_healthy + networks: + - platform-internal + - local-access + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/readyz"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 5s + postgres: image: postgres:17.9-bookworm@sha256:47f917f7409eacd22fc5dfb1dee634e1b55cf0c01d1a7eb701be2227a03e0641 environment: @@ -146,25 +175,34 @@ services: NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL: "${NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL:?NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL must be set and non-empty}" NEKIRO_ROUTER_AGENT_CREDENTIAL_TTL_SECONDS: "${NEKIRO_ROUTER_AGENT_CREDENTIAL_TTL_SECONDS:?NEKIRO_ROUTER_AGENT_CREDENTIAL_TTL_SECONDS must be set and non-empty}" NEKIRO_ROUTER_INSTANCE_ROUTING_MODE: "${NEKIRO_ROUTER_INSTANCE_ROUTING_MODE:?NEKIRO_ROUTER_INSTANCE_ROUTING_MODE must be set and non-empty}" - NEKIRO_ROUTER_NACOS_API_ORIGIN: "http://nacos:8848/nacos" + NEKIRO_ROUTER_NACOS_API_ORIGIN: "https://nacos-secure:9443/nacos" NEKIRO_ROUTER_NACOS_NAMESPACE_ID: "nekiro" NEKIRO_ROUTER_NACOS_CONFIG_GROUP: "NEKIRO" NEKIRO_ROUTER_NACOS_AUTH_MODE: "none" NEKIRO_ROUTER_NACOS_RESPONSE_LIMIT_BYTES: "${NEKIRO_ROUTER_NACOS_RESPONSE_LIMIT_BYTES:?NEKIRO_ROUTER_NACOS_RESPONSE_LIMIT_BYTES must be set and non-empty}" NEKIRO_ROUTER_NACOS_REQUEST_TIMEOUT_MS: "${NEKIRO_ROUTER_NACOS_REQUEST_TIMEOUT_MS:?NEKIRO_ROUTER_NACOS_REQUEST_TIMEOUT_MS must be set and non-empty}" + NEKIRO_ROUTER_NACOS_HTTP_TLS_CA_FILE: "/var/run/nekiro-nacos-tls/ca.pem" + NEKIRO_ROUTER_NACOS_HTTP_TLS_SERVER_NAME: "nacos.internal" + NEKIRO_ROUTER_NACOS_HTTP_TLS_CLIENT_CERT_FILE: "/var/run/nekiro-nacos-tls/client.pem" + NEKIRO_ROUTER_NACOS_HTTP_TLS_CLIENT_KEY_FILE: "/var/run/nekiro-nacos-tls/client-key.pem" NEKIRO_ROUTER_NACOS_OBSERVE_ENABLED: "true" - NEKIRO_ROUTER_NACOS_GRPC_TARGET: "nacos:9848" + NEKIRO_ROUTER_NACOS_GRPC_TARGET: "nacos-secure:9848" NEKIRO_ROUTER_NACOS_GRPC_CLIENT_IP: "172.28.0.10" NEKIRO_ROUTER_NACOS_GRPC_REQUEST_TIMEOUT_MS: "${NEKIRO_ROUTER_NACOS_GRPC_REQUEST_TIMEOUT_MS:?NEKIRO_ROUTER_NACOS_GRPC_REQUEST_TIMEOUT_MS must be set and non-empty}" NEKIRO_ROUTER_NACOS_PENDING_CHANGES: "${NEKIRO_ROUTER_NACOS_PENDING_CHANGES:?NEKIRO_ROUTER_NACOS_PENDING_CHANGES must be set and non-empty}" NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS: "${NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS:?NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS must be set and non-empty}" - NEKIRO_ROUTER_NACOS_GRPC_TRANSPORT_SECURITY: "insecure" + NEKIRO_ROUTER_NACOS_GRPC_TRANSPORT_SECURITY: "mtls" + NEKIRO_ROUTER_NACOS_GRPC_TLS_CA_FILE: "/var/run/nekiro-nacos-tls/ca.pem" + NEKIRO_ROUTER_NACOS_GRPC_TLS_SERVER_NAME: "nacos.internal" + NEKIRO_ROUTER_NACOS_GRPC_TLS_CLIENT_CERT_FILE: "/var/run/nekiro-nacos-tls/client.pem" + NEKIRO_ROUTER_NACOS_GRPC_TLS_CLIENT_KEY_FILE: "/var/run/nekiro-nacos-tls/client-key.pem" NEKIRO_ROUTER_CONFIG_CENTER_FILE_ROOT: "/var/lib/nekiro/config" NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES: "${NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES:?NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES must be set and non-empty}" NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY: "${NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY:?NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY must be set and non-empty}" NEKIRO_ROUTER_INSTANCE_PORT_NAME: "${NEKIRO_ROUTER_INSTANCE_PORT_NAME:?NEKIRO_ROUTER_INSTANCE_PORT_NAME must be set and non-empty}" volumes: - "${NEKIRO_ROUTER_CONFIG_CENTER_ROOT:?NEKIRO_ROUTER_CONFIG_CENTER_ROOT must be set and non-empty}:/var/lib/nekiro/config" + - "${NEKIRO_E2E_TLS_ROOT:?NEKIRO_E2E_TLS_ROOT must be set and non-empty}:/var/run/nekiro-nacos-tls:ro" ports: - "127.0.0.1:${A2A_ROUTER_PORT:?A2A_ROUTER_PORT must be set and non-empty}:8081" depends_on: @@ -174,6 +212,8 @@ services: condition: service_healthy nacos-bootstrap: condition: service_completed_successfully + nacos-secure: + condition: service_healthy networks: platform-internal: ipv4_address: 172.28.0.10 diff --git a/scripts/prepare.sh b/scripts/prepare.sh index 9e3bc24..23bfe4c 100755 --- a/scripts/prepare.sh +++ b/scripts/prepare.sh @@ -73,17 +73,20 @@ control_plane_image="nekiro-control-plane:${component_shas[core]}" router_image="nekiro-a2a-router:${component_shas[core]}" runtime_a_image="nekiro-runtime-a:${component_shas[samples]}" runtime_b_image="nekiro-runtime-b:${component_shas[samples]}" +secure_fixture_image="nekiro-nacos-secure-fixture:$(git rev-parse HEAD)" docker build --file "${component_dirs[core]}/apps/control-plane/Dockerfile" --tag "$control_plane_image" "${component_dirs[core]}" docker build --file "${component_dirs[core]}/apps/a2a-router/Dockerfile" --tag "$router_image" "${component_dirs[core]}" docker build --file "${component_dirs[samples]}/runtime-a/Dockerfile" --tag "$runtime_a_image" "${component_dirs[samples]}" docker build --file "${component_dirs[samples]}/runtime-b/Dockerfile" --tag "$runtime_b_image" "${component_dirs[samples]}" +docker build --file tests/fixtures/nacos-secure-fixture.Dockerfile --tag "$secure_fixture_image" . { printf 'NEKIRO_CONTROL_PLANE_IMAGE=%q\n' "$control_plane_image" printf 'NEKIRO_A2A_ROUTER_IMAGE=%q\n' "$router_image" printf 'NEKIRO_RUNTIME_A_IMAGE=%q\n' "$runtime_a_image" printf 'NEKIRO_RUNTIME_B_IMAGE=%q\n' "$runtime_b_image" + printf 'NEKIRO_NACOS_SECURE_PROXY_IMAGE=%q\n' "$secure_fixture_image" printf 'NEKIRO_STACK_CORE_DIR=%q\n' "${component_dirs[core]}" printf 'NEKIRO_STACK_CONSOLE_DIR=%q\n' "${component_dirs[console]}" printf 'NEKIRO_STACK_SDK_GO_DIR=%q\n' "${component_dirs[sdkGo]}" diff --git a/scripts/write-ci-env.sh b/scripts/write-ci-env.sh index bb8cc93..dc2ea88 100755 --- a/scripts/write-ci-env.sh +++ b/scripts/write-ci-env.sh @@ -60,6 +60,9 @@ NEKIRO_ROUTER_NACOS_PENDING_CHANGES=64 NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS=1024 NEKIRO_E2E_NACOS_TLS_PORT=19443 NEKIRO_E2E_NACOS_MTLS_PORT=19444 +NEKIRO_E2E_NACOS_ROUTER_HTTP_PORT=19445 +NEKIRO_E2E_NACOS_ROUTER_GRPC_PORT=19446 +NEKIRO_E2E_NACOS_FIXTURE_STATUS_PORT=19447 EOF printf 'NEKIRO_ROUTER_CONFIG_CENTER_ROOT=%s\n' "$config_root" >>"$output" printf 'NEKIRO_E2E_TLS_ROOT=%s\n' "$tls_root" >>"$output" @@ -96,6 +99,7 @@ NEKIRO_E2E_OWNER_TOKEN=acceptance-owner-token NEKIRO_E2E_USER_TOKEN=acceptance-user-token NEKIRO_E2E_OTHER_TOKEN=acceptance-other-token NEKIRO_E2E_DATABASE_URL=postgresql://nekiro_acceptance:acceptance-only-password@127.0.0.1:55432/nekiro_acceptance?sslmode=disable +NEKIRO_E2E_NACOS_FIXTURE_STATUS_URL=http://127.0.0.1:19447/status NEKIRO_ROUTER_INSTANCE_ROUTING_MODE=nacos NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY=router.nacos-bindings EOF @@ -158,6 +162,7 @@ NEKIRO_CONTROL_PLANE_IMAGE=nekiro-control-plane:compose-check NEKIRO_A2A_ROUTER_IMAGE=nekiro-a2a-router:compose-check NEKIRO_RUNTIME_A_IMAGE=nekiro-runtime-a:compose-check NEKIRO_RUNTIME_B_IMAGE=nekiro-runtime-b:compose-check +NEKIRO_NACOS_SECURE_PROXY_IMAGE=nekiro-nacos-secure-fixture:compose-check NEKIRO_ROUTER_INSTANCE_ROUTING_MODE=direct EOF ;; diff --git a/tests/backend/invoke_record_test.go b/tests/backend/invoke_record_test.go index eb12f01..9039a15 100644 --- a/tests/backend/invoke_record_test.go +++ b/tests/backend/invoke_record_test.go @@ -44,6 +44,7 @@ type acceptanceEnv struct { publicAgentOrigin string routerURL string nacosURL string + nacosFixtureStatus string routerToken string ownerToken string userToken string @@ -260,6 +261,7 @@ func TestInvokeToRecordAcceptance(t *testing.T) { assertFailureMatrix(t, client, env) assertConcurrentCalls(t, client, env) + assertRouterUsedSecureNacosBoundaries(t, client, env) assertStorageAndLogsAreMetadataOnly(t, env) } @@ -274,26 +276,45 @@ func loadAcceptanceEnv(t *testing.T) acceptanceEnv { t.Fatalf("NEKIRO_ENDPOINT_CHALLENGE_TTL_SECONDS must be an acceptance value from 2 through 15 seconds") } return acceptanceEnv{ - controlPlane: requiredEnv(t, "NEKIRO_E2E_CONTROL_PLANE_URL"), - publicAgentOrigin: requiredEnv(t, "NEKIRO_E2E_PUBLIC_AGENT_ORIGIN"), - routerURL: requiredEnv(t, "NEKIRO_E2E_ROUTER_URL"), - nacosURL: requiredEnv(t, "NEKIRO_E2E_NACOS_URL"), - routerToken: requiredEnv(t, "NEKIRO_E2E_ROUTER_TOKEN"), - ownerToken: requiredEnv(t, "NEKIRO_E2E_OWNER_TOKEN"), - userToken: requiredEnv(t, "NEKIRO_E2E_USER_TOKEN"), - otherToken: requiredEnv(t, "NEKIRO_E2E_OTHER_TOKEN"), - databaseURL: requiredEnv(t, "NEKIRO_E2E_DATABASE_URL"), - tlsRoot: requiredEnv(t, "NEKIRO_E2E_TLS_ROOT"), - tlsPort: requiredEnv(t, "NEKIRO_E2E_NACOS_TLS_PORT"), - mtlsPort: requiredEnv(t, "NEKIRO_E2E_NACOS_MTLS_PORT"), - composeFile: composeFile, - composeProject: requiredEnv(t, "NEKIRO_E2E_COMPOSE_PROJECT"), - challengeTTL: time.Duration(ttlSeconds) * time.Second, - credentialIssuer: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_ISSUER"), - credentialKeyID: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_KEY_ID"), - credentialPrivate: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL"), - releases: make(map[string]contracts.AgentReleaseResponse), - publicAgentIDs: make(map[string]string), + controlPlane: requiredEnv(t, "NEKIRO_E2E_CONTROL_PLANE_URL"), + publicAgentOrigin: requiredEnv(t, "NEKIRO_E2E_PUBLIC_AGENT_ORIGIN"), + routerURL: requiredEnv(t, "NEKIRO_E2E_ROUTER_URL"), + nacosURL: requiredEnv(t, "NEKIRO_E2E_NACOS_URL"), + nacosFixtureStatus: requiredEnv(t, "NEKIRO_E2E_NACOS_FIXTURE_STATUS_URL"), + routerToken: requiredEnv(t, "NEKIRO_E2E_ROUTER_TOKEN"), + ownerToken: requiredEnv(t, "NEKIRO_E2E_OWNER_TOKEN"), + userToken: requiredEnv(t, "NEKIRO_E2E_USER_TOKEN"), + otherToken: requiredEnv(t, "NEKIRO_E2E_OTHER_TOKEN"), + databaseURL: requiredEnv(t, "NEKIRO_E2E_DATABASE_URL"), + tlsRoot: requiredEnv(t, "NEKIRO_E2E_TLS_ROOT"), + tlsPort: requiredEnv(t, "NEKIRO_E2E_NACOS_TLS_PORT"), + mtlsPort: requiredEnv(t, "NEKIRO_E2E_NACOS_MTLS_PORT"), + composeFile: composeFile, + composeProject: requiredEnv(t, "NEKIRO_E2E_COMPOSE_PROJECT"), + challengeTTL: time.Duration(ttlSeconds) * time.Second, + credentialIssuer: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_ISSUER"), + credentialKeyID: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_KEY_ID"), + credentialPrivate: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL"), + releases: make(map[string]contracts.AgentReleaseResponse), + publicAgentIDs: make(map[string]string), + } +} + +func assertRouterUsedSecureNacosBoundaries(t *testing.T, client *http.Client, env acceptanceEnv) { + t.Helper() + result := doRequest(t, client, env.nacosFixtureStatus, http.MethodGet, "", "", nil) + var status struct { + HTTPRequests int64 `json:"httpRequests"` + GRPCConnections int64 `json:"grpcConnections"` + } + if result.status != http.StatusOK || json.Unmarshal(result.body, &status) != nil { + t.Fatalf("secure Nacos fixture status=%d body=%s", result.status, result.body) + } + if status.HTTPRequests < 2 { + t.Fatalf("Router did not complete both secure Config Center and Naming HTTP reads: %#v", status) + } + if status.GRPCConnections < 1 { + t.Fatalf("Router did not establish the secure Nacos gRPC watch: %#v", status) } } diff --git a/tests/backend/nacos_tls_fixture_test.go b/tests/backend/nacos_tls_fixture_test.go index d8df4d1..463a6ec 100644 --- a/tests/backend/nacos_tls_fixture_test.go +++ b/tests/backend/nacos_tls_fixture_test.go @@ -37,7 +37,7 @@ type secureNacosFixture struct { func startSecureNacosFixture(t *testing.T, env *acceptanceEnv) { t.Helper() - material := writeSecureNacosMaterial(t, env.tlsRoot) + material := loadSecureNacosMaterial(t, env.tlsRoot) fixture := &secureNacosFixture{} env.secureNacos = fixture target, err := url.Parse(env.nacosURL) @@ -78,6 +78,23 @@ func startSecureNacosFixture(t *testing.T, env *acceptanceEnv) { start(env.mtlsPort, true) } +func loadSecureNacosMaterial(t *testing.T, directory string) secureNacosMaterial { + t.Helper() + server, err := tls.LoadX509KeyPair(filepath.Join(directory, "server.pem"), filepath.Join(directory, "server-key.pem")) + if err != nil { + t.Fatalf("load generated secure Nacos server identity: %v", err) + } + caPEM, err := os.ReadFile(filepath.Join(directory, "ca.pem")) + if err != nil { + t.Fatalf("load generated secure Nacos CA: %v", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caPEM) { + t.Fatal("parse generated secure Nacos CA") + } + return secureNacosMaterial{server: server, caPool: pool} +} + func assertSecureRegistrationFailureMatrix(t *testing.T, env acceptanceEnv) { t.Helper() for name, test := range map[string]struct { diff --git a/tests/fixtures/nacos-secure-fixture.Dockerfile b/tests/fixtures/nacos-secure-fixture.Dockerfile new file mode 100644 index 0000000..1e77910 --- /dev/null +++ b/tests/fixtures/nacos-secure-fixture.Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.26.4-bookworm@sha256:b305420a68d0f229d91eb3b3ed9e519fcf2cf5461da4bef997bf927e8c0bfd2b AS build + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd/nacos-secure-fixture ./cmd/nacos-secure-fixture +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/nacos-secure-fixture ./cmd/nacos-secure-fixture + +FROM alpine:3.22 + +RUN apk add --no-cache ca-certificates wget \ + && adduser -D -H -u 10001 nekiro +COPY --from=build /out/nacos-secure-fixture /usr/local/bin/nacos-secure-fixture +USER nekiro +ENTRYPOINT ["/usr/local/bin/nacos-secure-fixture"] From cd8f039fd0f7015f6be20536f26d0e272fa6b644 Mon Sep 17 00:00:00 2001 From: Nene7ko_ <1604009816@qq.com> Date: Mon, 10 Aug 2026 21:29:17 +0800 Subject: [PATCH 2/2] fix(stack): isolate secure Nacos profile --- .env.example | 7 +++---- .github/workflows/ci.yml | 10 ++++------ README.md | 13 ++++++------- compose.router-nacos-secure.yaml | 19 +++++++++++++++++++ compose.yaml | 18 ++++-------------- scripts/write-ci-env.sh | 1 + tests/backend/invoke_record_test.go | 9 ++++++--- 7 files changed, 43 insertions(+), 34 deletions(-) create mode 100644 compose.router-nacos-secure.yaml diff --git a/.env.example b/.env.example index 5caf2d7..5c56e73 100644 --- a/.env.example +++ b/.env.example @@ -14,10 +14,9 @@ NEKIRO_RUNTIME_A_IMAGE=replace-with-prepared-image NEKIRO_RUNTIME_B_IMAGE=replace-with-prepared-image NEKIRO_NACOS_SECURE_PROXY_IMAGE=replace-with-prepared-image -# Select exactly one Router instance-routing mode. Use direct to dispatch to -# the exact published Card endpoint, or config_center_file to select an -# instance from the exact Release-scoped directory document. -NEKIRO_ROUTER_INSTANCE_ROUTING_MODE=replace-with-direct-or-config_center_file +# Select exactly one Router instance-routing mode: direct, config_center_file, +# or nacos. There is no automatic fallback between them. +NEKIRO_ROUTER_INSTANCE_ROUTING_MODE=replace-with-explicit-routing-mode NEKIRO_ROUTER_CONFIG_CENTER_ROOT=replace-with-absolute-host-directory NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES=replace-with-explicit-byte-limit NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY=replace-with-directory-key diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd278d4..355ce1b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,7 @@ jobs: shell: bash run: | docker compose --file compose.yaml config --quiet + docker compose --file compose.yaml --file compose.router-nacos-secure.yaml --profile router-nacos-secure config --quiet if grep -nE '^[[:space:]]+build:|image:[[:space:]].*(latest|:(main|master))([[:space:]"@]|$)' compose.yaml; then echo 'Compose contains a source build or floating image.' >&2 exit 1 @@ -125,7 +126,7 @@ jobs: run: go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" - name: Start fresh exact backend assembly - run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml up --detach --wait --wait-timeout 120 + run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml --file "$NEKIRO_E2E_COMPOSE_OVERRIDE_FILE" --profile router-nacos-secure up --detach --wait --wait-timeout 120 - name: Run trusted publication Invoke-to-Record acceptance run: go test -tags=e2e -count=1 ./tests/backend @@ -133,11 +134,11 @@ jobs: - name: Capture sanitized backend logs if: always() && steps.prepare.outcome == 'success' shell: bash - run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml --profile runtime-registration --profile watch-refresh logs --no-color 2>&1 | perl scripts/sanitize-logs.pl + run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml --file "$NEKIRO_E2E_COMPOSE_OVERRIDE_FILE" --profile router-nacos-secure --profile runtime-registration --profile watch-refresh logs --no-color 2>&1 | perl scripts/sanitize-logs.pl - name: Tear down backend assembly if: always() && steps.prepare.outcome == 'success' - run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml --profile runtime-registration --profile watch-refresh down --volumes --remove-orphans + run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml --file "$NEKIRO_E2E_COMPOSE_OVERRIDE_FILE" --profile router-nacos-secure --profile runtime-registration --profile watch-refresh down --volumes --remove-orphans browser: runs-on: ubuntu-latest @@ -181,9 +182,6 @@ jobs: done printf 'NEKIRO_STACK_PREPARED_ENV=%s\n' "$PREPARED_ENV" >>"$GITHUB_ENV" - - name: Generate ephemeral Nacos PKI - run: go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" - - name: Start fresh exact browser assembly run: docker compose --project-name "$NEKIRO_E2E_COMPOSE_PROJECT" --file compose.yaml up --detach --wait --wait-timeout 120 diff --git a/README.md b/README.md index a569526..d2865d1 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,12 @@ prepared image environment, then validate and start the stack: ```bash docker compose --file compose.yaml config --quiet -docker compose --project-name nekiro-stack --file compose.yaml up --detach --wait --wait-timeout 120 +go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" +docker compose --project-name nekiro-stack \ + --file compose.yaml \ + --file compose.router-nacos-secure.yaml \ + --profile router-nacos-secure \ + up --detach --wait --wait-timeout 120 go test -tags=e2e -count=1 ./tests/backend ``` @@ -94,12 +99,6 @@ with a wrong CA, wrong server name, or missing mTLS client identity must fail before publication and leave no routable instance. Test keys exist only in the caller-owned runtime directory and are never committed or recorded in Ledger. -For a local backend run, generate the ephemeral material before Compose starts: - -```bash -go run ./cmd/nacos-secure-fixture generate "$NEKIRO_E2E_TLS_ROOT" -``` - The secure fixture exposes only a status endpoint on the host. Its counters prove that the Router completed authenticated HTTP reads and established the authenticated gRPC watch; Nacos itself remains internal to Compose. diff --git a/compose.router-nacos-secure.yaml b/compose.router-nacos-secure.yaml new file mode 100644 index 0000000..922152f --- /dev/null +++ b/compose.router-nacos-secure.yaml @@ -0,0 +1,19 @@ +services: + a2a-router: + environment: + NEKIRO_ROUTER_NACOS_API_ORIGIN: "https://nacos-secure:9443/nacos" + NEKIRO_ROUTER_NACOS_HTTP_TLS_CA_FILE: "/var/run/nekiro-nacos-tls/ca.pem" + NEKIRO_ROUTER_NACOS_HTTP_TLS_SERVER_NAME: "nacos.internal" + NEKIRO_ROUTER_NACOS_HTTP_TLS_CLIENT_CERT_FILE: "/var/run/nekiro-nacos-tls/client.pem" + NEKIRO_ROUTER_NACOS_HTTP_TLS_CLIENT_KEY_FILE: "/var/run/nekiro-nacos-tls/client-key.pem" + NEKIRO_ROUTER_NACOS_GRPC_TARGET: "nacos-secure:9848" + NEKIRO_ROUTER_NACOS_GRPC_TRANSPORT_SECURITY: "mtls" + NEKIRO_ROUTER_NACOS_GRPC_TLS_CA_FILE: "/var/run/nekiro-nacos-tls/ca.pem" + NEKIRO_ROUTER_NACOS_GRPC_TLS_SERVER_NAME: "nacos.internal" + NEKIRO_ROUTER_NACOS_GRPC_TLS_CLIENT_CERT_FILE: "/var/run/nekiro-nacos-tls/client.pem" + NEKIRO_ROUTER_NACOS_GRPC_TLS_CLIENT_KEY_FILE: "/var/run/nekiro-nacos-tls/client-key.pem" + volumes: + - "${NEKIRO_E2E_TLS_ROOT:?NEKIRO_E2E_TLS_ROOT must be set and non-empty}:/var/run/nekiro-nacos-tls:ro" + depends_on: + nacos-secure: + condition: service_healthy diff --git a/compose.yaml b/compose.yaml index 28213a4..e1da71c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -41,6 +41,7 @@ services: restart: "no" nacos-secure: + profiles: ["router-nacos-secure"] image: "${NEKIRO_NACOS_SECURE_PROXY_IMAGE:?NEKIRO_NACOS_SECURE_PROXY_IMAGE must be set to the prepared immutable image}" command: ["serve"] environment: @@ -175,34 +176,25 @@ services: NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL: "${NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL:?NEKIRO_ROUTER_AGENT_CREDENTIAL_PRIVATE_KEY_BASE64URL must be set and non-empty}" NEKIRO_ROUTER_AGENT_CREDENTIAL_TTL_SECONDS: "${NEKIRO_ROUTER_AGENT_CREDENTIAL_TTL_SECONDS:?NEKIRO_ROUTER_AGENT_CREDENTIAL_TTL_SECONDS must be set and non-empty}" NEKIRO_ROUTER_INSTANCE_ROUTING_MODE: "${NEKIRO_ROUTER_INSTANCE_ROUTING_MODE:?NEKIRO_ROUTER_INSTANCE_ROUTING_MODE must be set and non-empty}" - NEKIRO_ROUTER_NACOS_API_ORIGIN: "https://nacos-secure:9443/nacos" + NEKIRO_ROUTER_NACOS_API_ORIGIN: "http://nacos:8848/nacos" NEKIRO_ROUTER_NACOS_NAMESPACE_ID: "nekiro" NEKIRO_ROUTER_NACOS_CONFIG_GROUP: "NEKIRO" NEKIRO_ROUTER_NACOS_AUTH_MODE: "none" NEKIRO_ROUTER_NACOS_RESPONSE_LIMIT_BYTES: "${NEKIRO_ROUTER_NACOS_RESPONSE_LIMIT_BYTES:?NEKIRO_ROUTER_NACOS_RESPONSE_LIMIT_BYTES must be set and non-empty}" NEKIRO_ROUTER_NACOS_REQUEST_TIMEOUT_MS: "${NEKIRO_ROUTER_NACOS_REQUEST_TIMEOUT_MS:?NEKIRO_ROUTER_NACOS_REQUEST_TIMEOUT_MS must be set and non-empty}" - NEKIRO_ROUTER_NACOS_HTTP_TLS_CA_FILE: "/var/run/nekiro-nacos-tls/ca.pem" - NEKIRO_ROUTER_NACOS_HTTP_TLS_SERVER_NAME: "nacos.internal" - NEKIRO_ROUTER_NACOS_HTTP_TLS_CLIENT_CERT_FILE: "/var/run/nekiro-nacos-tls/client.pem" - NEKIRO_ROUTER_NACOS_HTTP_TLS_CLIENT_KEY_FILE: "/var/run/nekiro-nacos-tls/client-key.pem" NEKIRO_ROUTER_NACOS_OBSERVE_ENABLED: "true" - NEKIRO_ROUTER_NACOS_GRPC_TARGET: "nacos-secure:9848" + NEKIRO_ROUTER_NACOS_GRPC_TARGET: "nacos:9848" NEKIRO_ROUTER_NACOS_GRPC_CLIENT_IP: "172.28.0.10" NEKIRO_ROUTER_NACOS_GRPC_REQUEST_TIMEOUT_MS: "${NEKIRO_ROUTER_NACOS_GRPC_REQUEST_TIMEOUT_MS:?NEKIRO_ROUTER_NACOS_GRPC_REQUEST_TIMEOUT_MS must be set and non-empty}" NEKIRO_ROUTER_NACOS_PENDING_CHANGES: "${NEKIRO_ROUTER_NACOS_PENDING_CHANGES:?NEKIRO_ROUTER_NACOS_PENDING_CHANGES must be set and non-empty}" NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS: "${NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS:?NEKIRO_ROUTER_NACOS_MAX_OBSERVATIONS must be set and non-empty}" - NEKIRO_ROUTER_NACOS_GRPC_TRANSPORT_SECURITY: "mtls" - NEKIRO_ROUTER_NACOS_GRPC_TLS_CA_FILE: "/var/run/nekiro-nacos-tls/ca.pem" - NEKIRO_ROUTER_NACOS_GRPC_TLS_SERVER_NAME: "nacos.internal" - NEKIRO_ROUTER_NACOS_GRPC_TLS_CLIENT_CERT_FILE: "/var/run/nekiro-nacos-tls/client.pem" - NEKIRO_ROUTER_NACOS_GRPC_TLS_CLIENT_KEY_FILE: "/var/run/nekiro-nacos-tls/client-key.pem" + NEKIRO_ROUTER_NACOS_GRPC_TRANSPORT_SECURITY: "insecure" NEKIRO_ROUTER_CONFIG_CENTER_FILE_ROOT: "/var/lib/nekiro/config" NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES: "${NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES:?NEKIRO_ROUTER_CONFIG_CENTER_MAX_PAYLOAD_BYTES must be set and non-empty}" NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY: "${NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY:?NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY must be set and non-empty}" NEKIRO_ROUTER_INSTANCE_PORT_NAME: "${NEKIRO_ROUTER_INSTANCE_PORT_NAME:?NEKIRO_ROUTER_INSTANCE_PORT_NAME must be set and non-empty}" volumes: - "${NEKIRO_ROUTER_CONFIG_CENTER_ROOT:?NEKIRO_ROUTER_CONFIG_CENTER_ROOT must be set and non-empty}:/var/lib/nekiro/config" - - "${NEKIRO_E2E_TLS_ROOT:?NEKIRO_E2E_TLS_ROOT must be set and non-empty}:/var/run/nekiro-nacos-tls:ro" ports: - "127.0.0.1:${A2A_ROUTER_PORT:?A2A_ROUTER_PORT must be set and non-empty}:8081" depends_on: @@ -212,8 +204,6 @@ services: condition: service_healthy nacos-bootstrap: condition: service_completed_successfully - nacos-secure: - condition: service_healthy networks: platform-internal: ipv4_address: 172.28.0.10 diff --git a/scripts/write-ci-env.sh b/scripts/write-ci-env.sh index dc2ea88..8adba38 100755 --- a/scripts/write-ci-env.sh +++ b/scripts/write-ci-env.sh @@ -105,6 +105,7 @@ NEKIRO_ROUTER_INSTANCE_DIRECTORY_KEY=router.nacos-bindings EOF printf 'NEKIRO_E2E_CONFIG_CENTER_ROOT=%s\n' "$config_root" >>"$output" printf 'NEKIRO_E2E_NACOS_URL=http://127.0.0.1:%s/nacos\n' "${NACOS_PORT:-18848}" >>"$output" + printf 'NEKIRO_E2E_COMPOSE_OVERRIDE_FILE=%s/compose.router-nacos-secure.yaml\n' "$stack_root" >>"$output" ;; browser) write_common diff --git a/tests/backend/invoke_record_test.go b/tests/backend/invoke_record_test.go index 9039a15..dd196cb 100644 --- a/tests/backend/invoke_record_test.go +++ b/tests/backend/invoke_record_test.go @@ -51,6 +51,7 @@ type acceptanceEnv struct { otherToken string databaseURL string composeFile string + composeOverride string composeProject string challengeTTL time.Duration credentialIssuer string @@ -268,8 +269,9 @@ func TestInvokeToRecordAcceptance(t *testing.T) { func loadAcceptanceEnv(t *testing.T) acceptanceEnv { t.Helper() composeFile := requiredEnv(t, "NEKIRO_E2E_COMPOSE_FILE") - if !filepath.IsAbs(composeFile) { - t.Fatalf("NEKIRO_E2E_COMPOSE_FILE must be an absolute path") + composeOverride := requiredEnv(t, "NEKIRO_E2E_COMPOSE_OVERRIDE_FILE") + if !filepath.IsAbs(composeFile) || !filepath.IsAbs(composeOverride) { + t.Fatalf("E2E Compose files must use absolute paths") } ttlSeconds, err := strconv.ParseInt(requiredEnv(t, "NEKIRO_ENDPOINT_CHALLENGE_TTL_SECONDS"), 10, 64) if err != nil || ttlSeconds < 2 || ttlSeconds > 15 { @@ -290,6 +292,7 @@ func loadAcceptanceEnv(t *testing.T) acceptanceEnv { tlsPort: requiredEnv(t, "NEKIRO_E2E_NACOS_TLS_PORT"), mtlsPort: requiredEnv(t, "NEKIRO_E2E_NACOS_MTLS_PORT"), composeFile: composeFile, + composeOverride: composeOverride, composeProject: requiredEnv(t, "NEKIRO_E2E_COMPOSE_PROJECT"), challengeTTL: time.Duration(ttlSeconds) * time.Second, credentialIssuer: requiredEnv(t, "NEKIRO_ROUTER_AGENT_CREDENTIAL_ISSUER"), @@ -545,7 +548,7 @@ func requiredEnv(t *testing.T, name string) string { } func composeCommand(ctx context.Context, env acceptanceEnv, args ...string) *exec.Cmd { - base := []string{"compose", "--project-name", env.composeProject, "--file", env.composeFile} + base := []string{"compose", "--project-name", env.composeProject, "--file", env.composeFile, "--file", env.composeOverride, "--profile", "router-nacos-secure"} command := exec.CommandContext(ctx, "docker", append(base, args...)...) command.Env = append(os.Environ(), runtimeRegistrationEnvironment(env)...) return command