diff --git a/changelog.d/security-19221-pin-git-sources-and-token-check.md b/changelog.d/security-19221-pin-git-sources-and-token-check.md new file mode 100644 index 0000000000..0b27d72d3c --- /dev/null +++ b/changelog.d/security-19221-pin-git-sources-and-token-check.md @@ -0,0 +1 @@ +- **Egress: git knowledge sources and the GitHub token check connect only to the address they checked** ([#19221](https://github.com/hivecommons/hive/issues/19221), part of [#18986](https://github.com/hivecommons/hive/issues/18986)). A git source's host was validated with one DNS lookup and then resolved again by `git` itself on every clone, `ls-remote` and pull, so a name that rebinds between the two was checked as public and fetched as private. Each git command now resolves the host once, refuses the fetch when any answer is private, loopback, link-local or otherwise barred, or when the lookup fails or comes back empty, and pins git to the checked addresses with `-c http.curloptResolve` (git 2.37 or later). Periodic syncs are checked the same way, so a source whose name later moves inward stops syncing. `HIVE_ALLOW_PRIVATE_GIT_SOURCE=true` still allows an internal Git server, but link-local (cloud metadata), multicast and unspecified answers stay refused. The dashboard's GitHub token check now dials through a pinned transport that refuses barred addresses for `api.github.com`, allows a configured API URL to be private while still refusing metadata answers, and refuses cross-host redirects. The hub's mail relay to a hive's self-reported dashboard URL now dials through the same pinned transport as the webhook relay. diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 4f73cc4a27..5c375c0dec 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -419,6 +419,23 @@ there. Operator-configured gateway endpoints and the fixed-host `/api/style` fetch are not pinned: a gateway may legitimately sit on a private address, and redirects off either are still checked. +The hub's contribute proxies, lite-enrollment repo access checks and mail +relay to a spoke's dashboard, and the dashboard's GitHub token check, dial +through the same pinned transport. The +token check refuses every barred class for public `api.github.com`; for a +configured API URL, which is intentionally chosen and may be private, it still +pins every dial but refuses only link-local (cloud metadata), multicast and +unspecified answers, and it refuses any cross-host redirect so that allowance +never reaches another host. Git knowledge sources run `git` as a subprocess, so +they pin through git instead: each clone, `ls-remote` and pull resolves the +host once, refuses the fetch when any answer is barred or the lookup fails or +comes back empty, and passes the checked addresses to git as +`-c http.curloptResolve=host:port:addr`, so git connects to them and never +looks the name up itself (git 2.37 or later; redirects are already refused with +`http.followRedirects=false`). `HIVE_ALLOW_PRIVATE_GIT_SOURCE=true` allows a +private or loopback Git server; link-local, multicast and unspecified answers +stay refused even then. + **Proxy decisions fail closed.** The egress audit ([#18986](https://github.com/hivecommons/hive/issues/18986), item 5) walked every branch of the proxy's per-request decision path that can fail and diff --git a/src/pkg/dashboard/api_contribute.go b/src/pkg/dashboard/api_contribute.go index 570a491870..0433467cbc 100644 --- a/src/pkg/dashboard/api_contribute.go +++ b/src/pkg/dashboard/api_contribute.go @@ -1737,14 +1737,22 @@ func validateGitHubToken(token, apiURL string) string { } ghTokenCacheMu.RUnlock() - userEndpoint := "https://api.github.com/user" - if apiURL != "" && apiURL != "https://api.github.com" { + userEndpoint := config.DefaultGitHubAPIURL + "/user" + ctx := context.Background() + if apiURL != "" && apiURL != config.DefaultGitHubAPIURL { userEndpoint = apiURL + "/user" + // A configured API URL is intentionally chosen and may be private; + // githubTokenTransport still pins and checks every dial to it. + ctx = context.WithValue(ctx, githubTokenConfiguredTargetKey{}, true) } const tokenValidateTimeout = 10 * time.Second - client := &http.Client{Timeout: tokenValidateTimeout} - req, err := http.NewRequest("GET", userEndpoint, nil) + client := &http.Client{ + Timeout: tokenValidateTimeout, + Transport: githubTokenRoundTripper(), + CheckRedirect: githubTokenCheckRedirect, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, userEndpoint, nil) if err != nil { return "" } diff --git a/src/pkg/dashboard/github_token_transport.go b/src/pkg/dashboard/github_token_transport.go new file mode 100644 index 0000000000..8bebe4897e --- /dev/null +++ b/src/pkg/dashboard/github_token_transport.go @@ -0,0 +1,107 @@ +package dashboard + +import ( + "context" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/netguard" +) + +const ( + // githubTokenDialTimeout and githubTokenDialKeepAlive match the + // net.Dialer http.DefaultTransport uses. + githubTokenDialTimeout = 30 * time.Second + githubTokenDialKeepAlive = 30 * time.Second + + // githubTokenMaxRedirects bounds a token check's same-host redirects. + githubTokenMaxRedirects = 3 +) + +// githubTokenConfiguredTargetKey marks a token-check request whose API URL is +// a configured override rather than public api.github.com. +type githubTokenConfiguredTargetKey struct{} + +// githubTokenConfiguredRefusedReasons are the address classes refused even +// for a configured API URL. A configured target is intentionally chosen and +// may be private or loopback (an httptest server, an internal mirror), but no +// GitHub API lives on link-local (cloud instance metadata), multicast or an +// unspecified address. +var githubTokenConfiguredRefusedReasons = map[string]bool{ + netguard.ReasonLinkLocal: true, + netguard.ReasonMulticast: true, + netguard.ReasonUnspecified: true, + netguard.ReasonThisNetwork: true, + netguard.ReasonBroadcast: true, +} + +// githubTokenDial connects to an address githubTokenTransport already +// checked. A package var so TestMain routes it through the suite's +// loopback-only guard and tests can observe where a dial would go. +var githubTokenDial netguard.DialFunc = (&net.Dialer{ + Timeout: githubTokenDialTimeout, + KeepAlive: githubTokenDialKeepAlive, +}).DialContext + +// githubTokenTransport is the transport for validateGitHubToken (#19221, +// part of #18986). The token check sends a user's bearer token to the API +// host, and a default http.Client resolved the name at dial time with nothing +// checking the answer. This transport resolves each dial once through +// privateURLResolver, checks every answer, fails closed when the lookup +// errors or returns nothing, and connects only to an address it checked. +// +// For public api.github.com every class isPrivateURL bars is refused. For a +// configured API URL (githubTokenConfiguredTargetKey on the request context) +// the target is intentionally chosen, so only +// githubTokenConfiguredRefusedReasons are refused; githubTokenCheckRedirect +// refuses a cross-host redirect, so that allowance never reaches another host. +var githubTokenTransport http.RoundTripper = netguard.PinnedDialer{ + Resolve: func(ctx context.Context, host string) ([]string, error) { return privateURLResolver(ctx, host) }, + Refuse: githubTokenRefuse, + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + return githubTokenDial(ctx, network, addr) + }, +}.Transport() + +func githubTokenRefuse(ctx context.Context, target string) bool { + configured, _ := ctx.Value(githubTokenConfiguredTargetKey{}).(bool) + if !configured { + return isPrivateURL(ctx, "http://"+target) + } + host, _, err := net.SplitHostPort(target) + if err != nil { + return true + } + addr, ok := netguard.ParseHostLiteral(host) + if !ok { + return true + } + return githubTokenConfiguredRefusedReasons[netguard.DenyReason(addr)] +} + +// githubTokenRoundTripper is githubTokenTransport, unless http.DefaultTransport +// was replaced with a non-*http.Transport RoundTripper (tests and +// instrumented builds inject one there, the same convention as the hub's +// pinnedOrReplacedDefault); then that replacement is used. +func githubTokenRoundTripper() http.RoundTripper { + if _, ok := http.DefaultTransport.(*http.Transport); ok { + return githubTokenTransport + } + return http.DefaultTransport +} + +// githubTokenCheckRedirect allows only a bounded number of same-host +// redirects. A cross-host hop would carry the configured-target allowance +// and the bearer token to a host nobody configured. +func githubTokenCheckRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= githubTokenMaxRedirects { + return fmt.Errorf("stopped after %d redirects", githubTokenMaxRedirects) + } + if len(via) > 0 && !strings.EqualFold(req.URL.Host, via[0].URL.Host) { + return fmt.Errorf("%w: cross-host redirect to %s", netguard.ErrRefused, req.URL.Host) + } + return nil +} diff --git a/src/pkg/dashboard/github_token_transport_test.go b/src/pkg/dashboard/github_token_transport_test.go new file mode 100644 index 0000000000..f91ffb1f1e --- /dev/null +++ b/src/pkg/dashboard/github_token_transport_test.go @@ -0,0 +1,194 @@ +package dashboard + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "sync/atomic" + "testing" + + "github.com/hivecommons/hive/pkg/netguard" +) + +// Resolve-once and dial pinning for validateGitHubToken (#19221, part of +// #18986). These tests drive a resolver that answers differently on later +// lookups and a recording dialer, so they never leave the machine. + +const ( + tokenPinPublicIP = "140.82.112.6" + tokenPinHost = "rebind.example" +) + +// stubTokenResolver answers first on the first lookup and later afterwards. +func stubTokenResolver(t *testing.T, first, later []string, err error) *int32 { + t.Helper() + var calls int32 + orig := privateURLResolver + privateURLResolver = func(_ context.Context, _ string) ([]string, error) { + if err != nil { + atomic.AddInt32(&calls, 1) + return nil, err + } + if atomic.AddInt32(&calls, 1) == 1 { + return first, nil + } + return later, nil + } + t.Cleanup(func() { privateURLResolver = orig }) + return &calls +} + +// recordTokenDials replaces githubTokenDial with one that records each +// target. When to is set, the dial connects there instead (an httptest +// server standing in for the checked public address). +func recordTokenDials(t *testing.T, to string) *[]string { + t.Helper() + var mu sync.Mutex + var got []string + orig := githubTokenDial + githubTokenDial = func(ctx context.Context, network, addr string) (net.Conn, error) { + mu.Lock() + got = append(got, addr) + mu.Unlock() + if to == "" { + return nil, errors.New("test dialer: no connection") + } + return (&net.Dialer{}).DialContext(ctx, network, to) + } + t.Cleanup(func() { githubTokenDial = orig }) + return &got +} + +// The invariant: a name that rebinds between the check and the dial cannot +// reach a private address. The pre-fetch check sees the public answer; the +// dial's own single lookup sees loopback and refuses, and the loopback server +// receives nothing. +func TestGitHubTokenTransport_RebindBetweenCheckAndDialRefused(t *testing.T) { + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&hits, 1) + })) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + stubTokenResolver(t, []string{tokenPinPublicIP}, []string{"127.0.0.1"}, nil) + // Real dialer, so a dial to the rebound address would reach srv. + orig := githubTokenDial + githubTokenDial = (&net.Dialer{}).DialContext + t.Cleanup(func() { githubTokenDial = orig }) + + target := "http://" + tokenPinHost + ":" + u.Port() + "/user" + if isPrivateURL(context.Background(), target) { + t.Fatal("pre-fetch check should see the public first answer") + } + req, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + resp, err := (&http.Client{Transport: githubTokenTransport}).Do(req) + if err == nil { + resp.Body.Close() + t.Fatal("dial after rebind succeeded") + } + if !errors.Is(err, netguard.ErrRefused) { + t.Fatalf("dial after rebind = %v; want netguard.ErrRefused", err) + } + if n := atomic.LoadInt32(&hits); n != 0 { + t.Fatalf("rebound loopback server received %d request(s)", n) + } +} + +// Default api.github.com: the dial connects to the address it checked, from +// one lookup, never to the name. +func TestValidateGitHubToken_DialsCheckedAddressOnce(t *testing.T) { + calls := stubTokenResolver(t, []string{tokenPinPublicIP}, []string{"127.0.0.1"}, nil) + dials := recordTokenDials(t, "") + + _ = validateGitHubToken("ghp_pin_checked_once", "") + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("resolver called %d times, want 1", got) + } + if len(*dials) != 1 || (*dials)[0] != net.JoinHostPort(tokenPinPublicIP, "443") { + t.Fatalf("dials = %v, want only the checked %s:443", *dials, tokenPinPublicIP) + } +} + +// Default api.github.com: a private answer (any of several), an empty answer +// or a lookup error refuses the dial before any connection is attempted. +func TestValidateGitHubToken_DefaultHostRefusesPrivateAndFailsClosed(t *testing.T) { + cases := []struct { + name string + answers []string + err error + }{ + {"loopback", []string{"127.0.0.1"}, nil}, + {"any answer private", []string{tokenPinPublicIP, "10.1.2.3"}, nil}, + {"metadata", []string{"169.254.169.254"}, nil}, + {"empty", nil, nil}, + {"lookup error", nil, errors.New("dns unavailable")}, + } + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stubTokenResolver(t, tc.answers, tc.answers, tc.err) + dials := recordTokenDials(t, "") + if got := validateGitHubToken("ghp_pin_refused_"+string(rune('a'+i)), ""); got != "" { + t.Fatalf("username = %q, want refusal", got) + } + if len(*dials) != 0 { + t.Fatalf("dialled %v despite a barred or failed lookup", *dials) + } + }) + } +} + +// A configured API URL is intentionally chosen and may be private: an +// httptest server on loopback answers. Link-local metadata stays refused. +func TestValidateGitHubToken_ConfiguredTargetMayBePrivateButNotMetadata(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"login":"octo-pin"}`)) + })) + t.Cleanup(srv.Close) + if got := validateGitHubToken("ghp_pin_configured_ok", srv.URL); got != "octo-pin" { + t.Fatalf("configured loopback API: username = %q, want octo-pin", got) + } + + stubTokenResolver(t, []string{"169.254.169.254"}, []string{"169.254.169.254"}, nil) + dials := recordTokenDials(t, "") + if got := validateGitHubToken("ghp_pin_configured_md", "http://ghe.internal.example"); got != "" { + t.Fatalf("configured host resolving to metadata: username = %q, want refusal", got) + } + if len(*dials) != 0 { + t.Fatalf("dialled %v for a metadata answer", *dials) + } +} + +// The configured-target allowance must not follow a cross-host redirect. +func TestValidateGitHubToken_CrossHostRedirectRefused(t *testing.T) { + var otherHits int32 + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&otherHits, 1) + _, _ = w.Write([]byte(`{"login":"elsewhere"}`)) + })) + t.Cleanup(other.Close) + ou, err := url.Parse(other.URL) + if err != nil { + t.Fatal(err) + } + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://localhost:"+ou.Port()+"/user", http.StatusFound) + })) + t.Cleanup(front.Close) + + if got := validateGitHubToken("ghp_pin_redirect", front.URL); got != "" { + t.Fatalf("username = %q after a cross-host redirect, want refusal", got) + } + if n := atomic.LoadInt32(&otherHits); n != 0 { + t.Fatalf("redirect target received %d request(s)", n) + } +} diff --git a/src/pkg/dashboard/main_test.go b/src/pkg/dashboard/main_test.go index 8d3687c548..d6e76ac7dd 100644 --- a/src/pkg/dashboard/main_test.go +++ b/src/pkg/dashboard/main_test.go @@ -66,6 +66,9 @@ func TestMain(m *testing.M) { } http.DefaultTransport = guarded http.DefaultClient.Transport = guarded + // validateGitHubToken dials through its own pinned transport (#19221); + // route its final connect through the same guard. + githubTokenDial = guarded.DialContext // Filesystem counterpart of the network guard above: newAuditLog wires a // lumberjack writer at the package-level auditLogPath (/data/audit.jsonl) diff --git a/src/pkg/hub/mail_relay.go b/src/pkg/hub/mail_relay.go index 28013fa6c8..6632294131 100644 --- a/src/pkg/hub/mail_relay.go +++ b/src/pkg/hub/mail_relay.go @@ -90,10 +90,16 @@ func mailChunkBytes() int64 { var mailDeliveryIDRE = regexp.MustCompile(`^[0-9a-f]{32}$`) // mailRelayClient never follows redirects, for the reason documented on -// spokeWebhookRelayClient. +// spokeWebhookRelayClient. Like that client, it dials only an address +// spokeWebhookRelayGuard passed, so a self-reported DashboardURL whose name +// rebinds between the guard in handleMailInbound and the dial is refused +// instead of reached (#19221). var mailRelayClient = &http.Client{ Timeout: mailRelayTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + Transport: pinnedFetchTransport(func(ctx context.Context, rawURL string) bool { + return spokeWebhookRelayGuard(ctx, rawURL) + }), } func (s *HubServer) registerMailRoutes() { diff --git a/src/pkg/hub/pinned_fetch_mail_test.go b/src/pkg/hub/pinned_fetch_mail_test.go new file mode 100644 index 0000000000..b33aa94959 --- /dev/null +++ b/src/pkg/hub/pinned_fetch_mail_test.go @@ -0,0 +1,36 @@ +package hub + +import ( + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/hivecommons/hive/pkg/mailin/wire" +) + +// The mail relay checks the self-reported DashboardURL with the real guard +// (first, public answer) and then posts to it. The post must resolve again +// through the same seam, see the rebound loopback answer and refuse it, so +// the loopback spoke receives nothing (#19221). +func TestMailRelayPinsResolvedAddress(t *testing.T) { + var reached atomic.Bool + spoke := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached.Store(true) })) + defer spoke.Close() + const host = "mail.rebind.example" + lookups := stubRebindingResolver(t, host) + s := mailTestHub(t, "http://"+host+":"+loopbackPort(t, spoke)+"/") + + rec := httptest.NewRecorder() + s.handleMailInbound(rec, mailRequest(http.MethodPost, wire.InboundPath, testMailToken, delivery(testDeliveryID, wire.KindInvite, "hive@acme.hive.hivecommons.dev"))) + + if reached.Load() { + t.Fatal("mail relay reached the rebound loopback address") + } + if rec.Code != http.StatusBadGateway { + t.Fatalf("rebound mail relay: status = %d, want 502", rec.Code) + } + if got := lookups(); got < 2 { + t.Fatalf("resolver consulted %d times, want the guard's lookup and the pinned dial's own", got) + } +} diff --git a/src/pkg/knowledge/gitsource.go b/src/pkg/knowledge/gitsource.go index 3958ab7b14..cd4a4502b5 100644 --- a/src/pkg/knowledge/gitsource.go +++ b/src/pkg/knowledge/gitsource.go @@ -15,6 +15,7 @@ import ( "time" "github.com/hivecommons/hive/crux/upstreamproxy" + "github.com/hivecommons/hive/pkg/netguard" ) const ( @@ -145,7 +146,7 @@ func (g *GitSource) Sync(ctx context.Context) error { return fmt.Errorf("clone dir %s is not a git repo", g.cloneDir) } - if err := gitSourcePull(ctx, g.cloneDir); err != nil { + if err := gitSourcePull(ctx, g.cloneDir, g.config.URL); err != nil { return fmt.Errorf("git source %s: pull failed: %w", g.config.Name, err) } @@ -188,7 +189,7 @@ func (g *GitSource) ensureCloned(ctx context.Context) error { "name", g.config.Name, "dir", g.cloneDir, ) - return gitSourcePull(ctx, g.cloneDir) + return gitSourcePull(ctx, g.cloneDir, g.config.URL) } if err := os.MkdirAll(filepath.Dir(g.cloneDir), 0o755); err != nil { @@ -206,7 +207,11 @@ func (g *GitSource) ensureCloned(ctx context.Context) error { args = append(args, "--", g.config.URL, g.cloneDir) - cmd := exec.CommandContext(cloneCtx, "git", gitCmdArgs(args...)...) + pin, err := gitSourcePinArgs(cloneCtx, g.config.URL) + if err != nil { + return fmt.Errorf("git clone: %w", err) + } + cmd := exec.CommandContext(cloneCtx, "git", gitCmdArgs(append(pin, args...)...)...) cmd.Env = gitSourceEnv() output, err := cmd.CombinedOutput() @@ -234,7 +239,11 @@ func (g *GitSource) ensureCloned(ctx context.Context) error { var gitSourceLsRemote = func(ctx context.Context, url string) ([]byte, error) { lsCtx, cancel := context.WithTimeout(ctx, gitSourceLookupTimeout*2) defer cancel() - cmd := exec.CommandContext(lsCtx, "git", gitCmdArgs("ls-remote", "--symref", "--", url, "HEAD")...) + pin, err := gitSourcePinArgs(lsCtx, url) + if err != nil { + return nil, err + } + cmd := exec.CommandContext(lsCtx, "git", gitCmdArgs(append(pin, "ls-remote", "--symref", "--", url, "HEAD")...)...) cmd.Env = gitSourceEnv() return cmd.CombinedOutput() } @@ -539,6 +548,105 @@ func gitCmdArgs(args ...string) []string { return append(append([]string{}, gitNoRedirectArgs...), args...) } +const ( + // gitHTTPSPort and gitHTTPPort are the ports git dials for a remote URL + // that names none. + gitHTTPSPort = "443" + gitHTTPPort = "80" + + // gitCurlResolveConfig is the git config key (git 2.37+) that hands + // libcurl a CURLOPT_RESOLVE entry, so curl connects to the listed + // addresses for host:port instead of looking the name up itself. + gitCurlResolveConfig = "http.curloptResolve" +) + +// gitSourceOptInRefusedReasons are the address classes refused even when the +// operator set HIVE_ALLOW_PRIVATE_GIT_SOURCE. The opt-in exists for an +// internal Git server on a private, loopback or carrier-grade NAT address; no +// Git server lives on link-local (cloud instance metadata), multicast or an +// unspecified address, so those stay refused. +var gitSourceOptInRefusedReasons = map[string]bool{ + netguard.ReasonLinkLocal: true, + netguard.ReasonMulticast: true, + netguard.ReasonUnspecified: true, + netguard.ReasonThisNetwork: true, + netguard.ReasonBroadcast: true, +} + +// gitSourceRefuse is the dial-time check for a resolved "ip:port" target. +// By default every class netguard names is refused; with the operator opt-in +// only gitSourceOptInRefusedReasons are. +func gitSourceRefuse(_ context.Context, target string) bool { + host, _, err := net.SplitHostPort(target) + if err != nil { + return true + } + addr, ok := netguard.ParseHostLiteral(host) + if !ok { + return true + } + reason := netguard.DenyReason(addr) + if allowPrivateGitSource() { + return gitSourceOptInRefusedReasons[reason] + } + return reason != "" +} + +// gitSourcePinArgs closes the rebinding window between validating a git +// source URL and git connecting to it (#19221, part of #18986). +// ValidateGitSourceURLContext resolves the host, and git (libcurl) used to +// resolve it again on every clone, ls-remote and pull, so a name whose +// answer changed between the two lookups was checked as public and fetched +// as private. This resolves the host once through gitSourceResolver, refuses +// the fetch when any answer is barred or the lookup fails or comes back empty +// (fail closed), and returns the leading `-c http.curloptResolve=...` override +// that makes curl connect only to the addresses it checked. Redirects are +// already refused by gitNoRedirectArgs, so the pinned host is the only host +// the command talks to. +// +// An IP-literal host needs no pin (curl does not resolve it); it is still +// classified here. Behind an upstream forward proxy (gitSourceEnv) curl +// connects to the proxy and the proxy resolves the target, the same documented +// limit as netguard.PinnedDialer.Transport. +func gitSourcePinArgs(ctx context.Context, rawURL string) ([]string, error) { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || parsed.Hostname() == "" { + return nil, fmt.Errorf("%w: git URL host is required", netguard.ErrRefused) + } + host := strings.ToLower(parsed.Hostname()) + port := parsed.Port() + if port == "" { + port = gitHTTPSPort + if strings.EqualFold(parsed.Scheme, "http") { + port = gitHTTPPort + } + } + lookupCtx, cancel := context.WithTimeout(ctx, gitSourceLookupTimeout) + defer cancel() + targets, err := netguard.PinnedDialer{ + Resolve: func(ctx context.Context, h string) ([]string, error) { return gitSourceResolver(ctx, h) }, + Refuse: gitSourceRefuse, + }.Targets(lookupCtx, net.JoinHostPort(host, port)) + if err != nil { + return nil, err + } + if _, literal := netguard.ParseHostLiteral(host); literal { + return nil, nil + } + addrs := make([]string, 0, len(targets)) + for _, t := range targets { + ip, _, err := net.SplitHostPort(t) + if err != nil { + return nil, fmt.Errorf("%w: %v", netguard.ErrRefused, err) + } + if strings.Contains(ip, ":") { + ip = "[" + ip + "]" + } + addrs = append(addrs, ip) + } + return []string{"-c", gitCurlResolveConfig + "=" + host + ":" + port + ":" + strings.Join(addrs, ",")}, nil +} + func gitSourceEnv() []string { allowedProtocols := gitAllowedProtocols if allowPrivateGitSource() { diff --git a/src/pkg/knowledge/gitsource_f8_redirect_test.go b/src/pkg/knowledge/gitsource_f8_redirect_test.go index 87141b231a..4ff274cab2 100644 --- a/src/pkg/knowledge/gitsource_f8_redirect_test.go +++ b/src/pkg/knowledge/gitsource_f8_redirect_test.go @@ -176,7 +176,7 @@ func TestF8GitPullRefusesRedirect(t *testing.T) { defer cancel() // The pull is expected to fail; what matters is WHERE it did not go. - _ = gitSourcePull(ctx, repo) + _ = gitSourcePull(ctx, repo, remote) if *frontHits == 0 { t.Fatal("test never reached the validated remote — no redirect was exercised") diff --git a/src/pkg/knowledge/gitsource_pin_test.go b/src/pkg/knowledge/gitsource_pin_test.go new file mode 100644 index 0000000000..4c27bcc3d4 --- /dev/null +++ b/src/pkg/knowledge/gitsource_pin_test.go @@ -0,0 +1,204 @@ +package knowledge + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os/exec" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/netguard" +) + +// Resolve-once and dial pinning for git knowledge sources (#19221, part of +// #18986). ValidateGitSourceURLContext resolves the host, and git used to +// resolve it again itself, so a name that answers public on the first lookup +// and private on the second was validated as public and fetched as private. +// These tests drive a resolver that answers differently on later lookups. + +const ( + pinTestPublicIP = "93.184.216.34" + pinTestLoopbackIP = "127.0.0.1" + pinTestHost = "rebind.example" + + // gitCurlResolveMinMajor / Minor is the first git with http.curloptResolve. + gitCurlResolveMinMajor = 2 + gitCurlResolveMinMinor = 37 +) + +// stubRebindingGitResolver answers first on the first lookup and later on +// every lookup after it, and counts lookups. +func stubRebindingGitResolver(t *testing.T, first, later []string) *int32 { + t.Helper() + var calls int32 + orig := gitSourceResolver + gitSourceResolver = func(_ context.Context, _ string) ([]string, error) { + if atomic.AddInt32(&calls, 1) == 1 { + return first, nil + } + return later, nil + } + t.Cleanup(func() { gitSourceResolver = orig }) + return &calls +} + +func stubGitResolver(t *testing.T, answers []string, err error) { + t.Helper() + orig := gitSourceResolver + gitSourceResolver = func(_ context.Context, _ string) ([]string, error) { return answers, err } + t.Cleanup(func() { gitSourceResolver = orig }) +} + +// The invariant: a name that rebinds between validation and the git command +// cannot reach a private address. Validation sees the public answer; the pin +// lookup sees loopback and refuses, so git is never started. +func TestGitSourcePin_RebindBetweenValidateAndFetchRefused(t *testing.T) { + t.Setenv("HIVE_ALLOW_PRIVATE_GIT_SOURCE", "") + stubRebindingGitResolver(t, []string{pinTestPublicIP}, []string{pinTestLoopbackIP}) + + raw := "https://" + pinTestHost + "/repo.git" + if err := ValidateGitSourceURL(raw); err != nil { + t.Fatalf("validation should see the public first answer: %v", err) + } + args, err := gitSourcePinArgs(context.Background(), raw) + if !errors.Is(err, netguard.ErrRefused) { + t.Fatalf("pin after rebind = %v, %v; want netguard.ErrRefused", args, err) + } + // The same refusal surfaces from the ls-remote path, before git runs. + if _, err := gitSourceLsRemote(context.Background(), raw); !errors.Is(err, netguard.ErrRefused) { + t.Fatalf("ls-remote after rebind = %v; want netguard.ErrRefused", err) + } +} + +// The pin names exactly the address that was checked, from one lookup, so +// curl has no name left to resolve. +func TestGitSourcePin_PinsCheckedAddressFromOneLookup(t *testing.T) { + t.Setenv("HIVE_ALLOW_PRIVATE_GIT_SOURCE", "") + calls := stubRebindingGitResolver(t, []string{pinTestPublicIP}, []string{pinTestLoopbackIP}) + + args, err := gitSourcePinArgs(context.Background(), "https://Rebind.Example/repo.git") + if err != nil { + t.Fatalf("pin: %v", err) + } + want := []string{"-c", gitCurlResolveConfig + "=" + pinTestHost + ":443:" + pinTestPublicIP} + if strings.Join(args, " ") != strings.Join(want, " ") { + t.Fatalf("pin args = %q, want %q", args, want) + } + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("resolver called %d times, want exactly 1", got) + } +} + +func TestGitSourcePin_RefusesBarredOrUnresolvable(t *testing.T) { + t.Setenv("HIVE_ALLOW_PRIVATE_GIT_SOURCE", "") + cases := []struct { + name string + answers []string + err error + }{ + {"any answer private", []string{pinTestPublicIP, "10.0.0.5"}, nil}, + {"metadata", []string{"169.254.169.254"}, nil}, + {"loopback v6", []string{"::1"}, nil}, + {"empty answer", nil, nil}, + {"unparseable answer", []string{"not-an-ip"}, nil}, + {"lookup error", nil, errors.New("dns unavailable")}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stubGitResolver(t, tc.answers, tc.err) + if args, err := gitSourcePinArgs(context.Background(), "https://"+pinTestHost+"/r.git"); !errors.Is(err, netguard.ErrRefused) { + t.Fatalf("got %q, %v; want netguard.ErrRefused", args, err) + } + }) + } +} + +// With the operator opt-in an internal Git server is intentionally private: +// it is allowed and still pinned. Cloud metadata stays refused. +func TestGitSourcePin_OptInAllowsPrivateButNotMetadata(t *testing.T) { + t.Setenv("HIVE_ALLOW_PRIVATE_GIT_SOURCE", "true") + + stubGitResolver(t, []string{"10.0.0.5", "fd00::5"}, nil) + args, err := gitSourcePinArgs(context.Background(), "http://git.internal.corp:8080/r.git") + if err != nil { + t.Fatalf("opt-in private pin: %v", err) + } + want := gitCurlResolveConfig + "=git.internal.corp:8080:10.0.0.5,[fd00::5]" + if len(args) != 2 || args[1] != want { + t.Fatalf("pin args = %q, want -c %q", args, want) + } + + stubGitResolver(t, []string{"169.254.169.254"}, nil) + if _, err := gitSourcePinArgs(context.Background(), "https://git.internal.corp/r.git"); !errors.Is(err, netguard.ErrRefused) { + t.Fatalf("opt-in must still refuse link-local metadata, got %v", err) + } +} + +func TestGitSourcePin_LiteralHost(t *testing.T) { + t.Setenv("HIVE_ALLOW_PRIVATE_GIT_SOURCE", "") + stubGitResolver(t, nil, errors.New("a literal must not be looked up")) + if args, err := gitSourcePinArgs(context.Background(), "https://"+pinTestPublicIP+"/r.git"); err != nil || args != nil { + t.Fatalf("public literal = %q, %v; want no pin and no error", args, err) + } + if _, err := gitSourcePinArgs(context.Background(), "https://0x7f.1/r.git"); !errors.Is(err, netguard.ErrRefused) { + t.Fatalf("loopback literal in hex spelling must be refused, got %v", err) + } +} + +func gitSupportsCurlResolve(t *testing.T) bool { + t.Helper() + out, err := exec.Command("git", "version").Output() + if err != nil { + return false + } + fields := strings.Fields(string(out)) + if len(fields) < 3 { + return false + } + parts := strings.SplitN(fields[2], ".", 3) + if len(parts) < 2 { + return false + } + major, err1 := strconv.Atoi(parts[0]) + minor, err2 := strconv.Atoi(parts[1]) + if err1 != nil || err2 != nil { + return false + } + return major > gitCurlResolveMinMajor || (major == gitCurlResolveMinMajor && minor >= gitCurlResolveMinMinor) +} + +// End to end: git honours the pin. The host is under .invalid, which system +// DNS never resolves, so the local server is reached only if git connected to +// the pinned address instead of looking the name up itself. +func TestGitSourcePin_GitConnectsToPinnedAddress(t *testing.T) { + f8RequireGit(t) + if !gitSupportsCurlResolve(t) { + t.Skip("git older than 2.37 has no http.curloptResolve") + } + t.Setenv("HIVE_ALLOW_PRIVATE_GIT_SOURCE", "true") + + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + http.NotFound(w, r) + })) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + stubGitResolver(t, []string{pinTestLoopbackIP}, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, _ = gitSourceLsRemote(ctx, "http://pinned-git.invalid:"+u.Port()+"/repo.git") + if atomic.LoadInt32(&hits) == 0 { + t.Fatal("git never reached the pinned address: the resolve pin was not applied") + } +} diff --git a/src/pkg/knowledge/gitsync.go b/src/pkg/knowledge/gitsync.go index b8fd89547a..15415ca138 100644 --- a/src/pkg/knowledge/gitsync.go +++ b/src/pkg/knowledge/gitsync.go @@ -15,6 +15,9 @@ const ( // gitDirName is the hidden directory that marks a git repository. gitDirName = ".git" + + // gitPullTimeout bounds one `git pull`, the resolve pin included. + gitPullTimeout = 30 * time.Second ) // GitSyncer periodically runs `git pull` on vault directories that are @@ -103,19 +106,31 @@ func gitPull(ctx context.Context, dir string) error { return gitPullWithEnv(ctx, dir, append(os.Environ(), "GIT_TERMINAL_PROMPT=0")) } -func gitSourcePull(ctx context.Context, dir string) error { - return gitPullWithEnv(ctx, dir, gitSourceEnv()) +// gitSourcePull pulls a knowledge-source clone from remoteURL, the source's +// configured URL. The remote's host is resolved once and the pull is pinned +// to the checked addresses (gitSourcePinArgs), so a name that rebinds to a +// private address after the source was validated is refused at sync time +// rather than fetched. +func gitSourcePull(ctx context.Context, dir, remoteURL string) error { + pullCtx, cancel := context.WithTimeout(ctx, gitPullTimeout) + defer cancel() + pin, err := gitSourcePinArgs(pullCtx, remoteURL) + if err != nil { + return &gitPullError{dir: dir, err: err} + } + return gitPullWithEnv(pullCtx, dir, gitSourceEnv(), pin...) } -func gitPullWithEnv(ctx context.Context, dir string, env []string) error { - const gitPullTimeoutSeconds = 30 - pullCtx, cancel := context.WithTimeout(ctx, gitPullTimeoutSeconds*time.Second) +// gitPullWithEnv runs `git pull --ff-only` in dir. extra are leading `-c` +// overrides (a resolve pin) placed before the subcommand. +func gitPullWithEnv(ctx context.Context, dir string, env []string, extra ...string) error { + pullCtx, cancel := context.WithTimeout(ctx, gitPullTimeout) defer cancel() // AUDIT F8 (residual): `--ff-only` bounds what git will MERGE, not where it // will FETCH FROM. Without redirect suppression the remote can 302 this pull // at an internal address after the URL passed validation. See gitNoRedirectArgs. - cmd := exec.CommandContext(pullCtx, "git", gitCmdArgs("pull", "--ff-only")...) + cmd := exec.CommandContext(pullCtx, "git", gitCmdArgs(append(append([]string{}, extra...), "pull", "--ff-only")...)...) cmd.Dir = dir cmd.Env = env