diff --git a/.github/workflows/stack-cloud.yaml b/.github/workflows/stack-cloud.yaml index 5def093..5420092 100644 --- a/.github/workflows/stack-cloud.yaml +++ b/.github/workflows/stack-cloud.yaml @@ -4,6 +4,9 @@ jobs: build-and-test: name: Build and Test runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v2 - uses: haskell-actions/setup@v2 @@ -11,8 +14,21 @@ jobs: enable-stack: true stack-no-global: true stack-version: "3.3.1" + - id: auth + name: Authenticate to Google Cloud + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: projects/3367369000/locations/global/workloadIdentityPools/wip-github-yqou/providers/wip-provider-github + service_account: sa-ci-cloud-pubsub@proda-ci.iam.gserviceaccount.com + create_credentials_file: true + export_environment_variables: true + universe: googleapis.com + cleanup_credentials: true + access_token_lifetime: 3600s + access_token_scopes: https://www.googleapis.com/auth/cloud-platform + id_token_include_email: false - name: Cache Stack Dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: | ~/.stack @@ -23,12 +39,7 @@ jobs: - ${{ runner.os }}- - name: Install Dependencies run: stack install --only-dependencies --test - - run: 'mkdir secrets && echo "$GCP_SA_KEY" > ./secrets/service_account.json' - shell: bash - env: - GCP_SA_KEY: ${{secrets.GCP_SA_KEY}} - name: Run Tests run: stack test env: PROJECT_ID: ${{secrets.PROJECT_ID}} - GOOGLE_APPLICATION_CREDENTIALS: ./secrets/service_account.json diff --git a/package.yaml b/package.yaml index d310e0d..8ba780b 100644 --- a/package.yaml +++ b/package.yaml @@ -115,6 +115,7 @@ tests: - async - bytestring - cloud-pubsub + - directory - exceptions - hspec # See https://github.com/input-output-hk/haskell.nix/issues/231 diff --git a/set-env.sh b/set-env.sh index 81924b0..cbf6869 100644 --- a/set-env.sh +++ b/set-env.sh @@ -10,8 +10,12 @@ if [[ -z "$PUBSUB_EMULATOR_HOST" ]] && [[ -z "$GOOGLE_APPLICATION_CREDENTIALS" ] PUBSUB_EMULATOR_HOST="localhost:8085" fi -# To run tests against hosted Cloud PubSub set GOOGLE_APPLICATION_CREDENTIALS -# instead of PUBSUB_EMULATOR_HOST -# GOOGLE_APPLICATION_CREDENTIALS="secrets/service_account.json" +# To run tests against hosted Cloud PubSub: +# Option 1 (recommended for local): Use gcloud auth application-default login +# This creates credentials at ~/.config/gcloud/application_default_credentials.json +# No need to set GOOGLE_APPLICATION_CREDENTIALS +# Option 2 (legacy): Set GOOGLE_APPLICATION_CREDENTIALS to a service account key file +# GOOGLE_APPLICATION_CREDENTIALS="secrets/service_account.json" +# Option 3 (GCP VM/CI): Don't set GOOGLE_APPLICATION_CREDENTIALS, use metadata server set +o allexport diff --git a/src/Cloud/PubSub.hs b/src/Cloud/PubSub.hs index 56a8e8e..b74583f 100644 --- a/src/Cloud/PubSub.hs +++ b/src/Cloud/PubSub.hs @@ -24,6 +24,7 @@ import qualified Network.HTTP.Client.TLS as HttpClientTLS data AuthMethod = ServiceAccountFile FilePath | MetadataServer + | ApplicationDefaultCredentialsFile FilePath deriving (Show, Eq) newtype HostAndPort = HostAndPort { unwrapHostAndPort :: String } @@ -53,6 +54,8 @@ mkClientResources projectId target = do AuthT.FromServiceAccount <$> Auth.readServiceAccountFile saFile MetadataServer -> pure AuthT.FromMetadataServer + ApplicationDefaultCredentialsFile adcFile -> + AuthT.FromApplicationDefaultCredentials <$> Auth.readApplicationDefaultCredentialsFile adcFile let resources = CloudTargetResources authSource tokenVar threshold serviceUrl = "https://pubsub.googleapis.com" return (serviceUrl, Cloud resources) diff --git a/src/Cloud/PubSub/Auth.hs b/src/Cloud/PubSub/Auth.hs index 9445b4b..ac0f7b3 100644 --- a/src/Cloud/PubSub/Auth.hs +++ b/src/Cloud/PubSub/Auth.hs @@ -2,6 +2,9 @@ module Cloud.PubSub.Auth ( fetchToken , readServiceAccountFile , fetchMetadataToken + , readApplicationDefaultCredentialsFile + , fetchApplicationDefaultCredentialsToken + , ADCCredentials(..) ) where import qualified Cloud.PubSub.Auth.Types as AuthT @@ -11,10 +14,12 @@ import Crypto.PubKey.RSA ( PrivateKey ) import qualified Crypto.PubKey.RSA.PKCS15 as RSA import qualified Data.Aeson as Aeson import Data.Aeson ( (.=) ) +import qualified Data.HashMap.Strict as HashMap import Data.ByteString ( ByteString ) import qualified Data.ByteString.Base64.URL as Base64 import qualified Data.ByteString.Lazy as LBS import Data.Functor ( (<&>) ) +import GHC.Generics ( Generic ) import qualified Data.Text as Text import qualified Data.Text.Encoding as TE import qualified Data.Time as Time @@ -25,6 +30,48 @@ readServiceAccountFile :: MonadIO m => FilePath -> m AuthT.ServiceAccount readServiceAccountFile fp = liftIO $ Aeson.eitherDecodeFileStrict fp >>= either fail return +data ADCCredentials = ADCCredentials + { adcType :: Text.Text + , adcClientId :: Text.Text + , adcClientSecret :: Text.Text + , adcRefreshToken :: Text.Text + } + deriving stock (Generic, Show) + +instance Aeson.FromJSON ADCCredentials where + parseJSON = Aeson.genericParseJSON $ Aeson.defaultOptions + { Aeson.fieldLabelModifier = Aeson.camelTo2 '_' . drop 3 + } + +readApplicationDefaultCredentialsFile :: MonadIO m => FilePath -> m AuthT.ApplicationDefaultCredentials +readApplicationDefaultCredentialsFile fp = liftIO $ do + result <- Aeson.eitherDecodeFileStrict fp :: IO (Either String ADCCredentials) + case result of + Right adc | adcType adc == Text.pack "authorized_user" -> + return $ AuthT.ApplicationDefaultCredentials + { AuthT.adcClientId = adcClientId adc + , AuthT.adcClientSecret = adcClientSecret adc + , AuthT.adcRefreshToken = adcRefreshToken adc + } + Right adc -> + fail $ "Unsupported ADC credentials type: " <> Text.unpack (adcType adc) <> + ". Only 'authorized_user' type is supported. For WIF/impersonated service accounts," <> + " please use environment variables or a supported credential format." + Left err -> do + -- Try to read as raw JSON to provide better error message + rawResult <- Aeson.eitherDecodeFileStrict fp :: IO (Either String Aeson.Value) + case rawResult of + Right (Aeson.Object obj) -> + case HashMap.lookup (Text.pack "type") obj of + Just (Aeson.String credType) -> + fail $ "Unsupported credential type: " <> Text.unpack credType <> + ". Expected 'authorized_user' ADC format with client_id, client_secret, and refresh_token fields." + _ -> + fail $ "Could not parse credentials file. Expected ADC format with 'authorized_user' type. " <> + "Parse error: " <> err + _ -> + fail $ "Could not parse credentials file as JSON. Parse error: " <> err + createAssertionTokenBody :: AuthT.PrivateKeyId -> AuthT.TokenClaims -> ByteString createAssertionTokenBody (AuthT.PrivateKeyId keyId) tokenClaims = @@ -107,3 +154,29 @@ fetchMetadataToken manager scope = liftIO $ do ) response <- HTTP.httpJSON request return $ HTTP.getResponseBody response + +-- | Fetches a GCP access token using Application Default Credentials (from gcloud auth application-default login) +-- Implements the OAuth2 refresh token flow described here: +-- https://developers.google.com/identity/protocols/oauth2/web-server#offline +fetchApplicationDefaultCredentialsToken + :: MonadIO m + => HttpClientC.Manager + -> AuthT.ApplicationDefaultCredentials + -> AuthT.Scope + -> m AuthT.AccessTokenResponse +fetchApplicationDefaultCredentialsToken manager adc scope = liftIO $ do + let tokenUrl = "https://oauth2.googleapis.com/token" + formData = + [ ("client_id" , TE.encodeUtf8 $ AuthT.adcClientId adc) + , ("client_secret", TE.encodeUtf8 $ AuthT.adcClientSecret adc) + , ("refresh_token" , TE.encodeUtf8 $ AuthT.adcRefreshToken adc) + , ("grant_type" , "refresh_token") + , ("scope" , TE.encodeUtf8 $ AuthT.unwrapScope scope) + ] + request <- + HTTP.parseRequest ("POST " <> tokenUrl) + <&> ( HTTP.setRequestBodyURLEncoded formData + . HTTP.setRequestManager manager + ) + response <- HTTP.httpJSON request + return $ HTTP.getResponseBody response diff --git a/src/Cloud/PubSub/Auth/Token.hs b/src/Cloud/PubSub/Auth/Token.hs index 5a24816..4a03641 100644 --- a/src/Cloud/PubSub/Auth/Token.hs +++ b/src/Cloud/PubSub/Auth/Token.hs @@ -28,6 +28,8 @@ fetchAndUpdateToken resources manager scope = do Auth.fetchToken manager serviceAccount scope Auth.FromMetadataServer -> Auth.fetchMetadataToken manager scope + Auth.FromApplicationDefaultCredentials adc -> + Auth.fetchApplicationDefaultCredentialsToken manager adc scope now <- Time.getCurrentTime let accessToken = tokenResponse.accessToken expiresAt = Time.addUTCTime tokenResponse.expiresIn now diff --git a/src/Cloud/PubSub/Auth/Types.hs b/src/Cloud/PubSub/Auth/Types.hs index 095cec5..f661c83 100644 --- a/src/Cloud/PubSub/Auth/Types.hs +++ b/src/Cloud/PubSub/Auth/Types.hs @@ -7,6 +7,7 @@ module Cloud.PubSub.Auth.Types , Scope(..) , ServiceAccount(..) , TokenSource(..) + , ApplicationDefaultCredentials(..) , TokenClaims(..) , UnixEpochSeconds(..) , X509PrivateKey(..) @@ -62,8 +63,15 @@ instance Aeson.FromJSON ServiceAccount where { Aeson.fieldLabelModifier = Aeson.camelTo2 '_' . drop 2 } +data ApplicationDefaultCredentials = ApplicationDefaultCredentials + { adcClientId :: Text + , adcClientSecret :: Text + , adcRefreshToken :: Text + } + data TokenSource = FromServiceAccount ServiceAccount | FromMetadataServer + | FromApplicationDefaultCredentials ApplicationDefaultCredentials newtype AccessToken = AccessToken { unwrapAccessToken :: Text diff --git a/test/Cloud/PubSub/AuthSpec.hs b/test/Cloud/PubSub/AuthSpec.hs index 840066b..8680f51 100644 --- a/test/Cloud/PubSub/AuthSpec.hs +++ b/test/Cloud/PubSub/AuthSpec.hs @@ -2,6 +2,8 @@ module Cloud.PubSub.AuthSpec where import qualified Cloud.PubSub.Auth as Auth import qualified Cloud.PubSub.Auth.Types as AuthT +import qualified Data.Aeson as Aeson +import qualified Data.Text as Text import qualified Network.HTTP.Client as HttpClient import qualified System.Environment as SystemEnv import qualified Network.HTTP.Client.TLS as HttpClientTLS @@ -10,13 +12,30 @@ import Test.Hspec tokenGetTest :: IO () tokenGetTest = do manager <- HttpClient.newManager HttpClientTLS.tlsManagerSettings - SystemEnv.lookupEnv "GOOGLE_APPLICATION_CREDENTIALS" >>= \case - Nothing -> pendingWith "skipping as auth not supported in emulator" - Just serviceAccountPath -> do - serviceAccount <- Auth.readServiceAccountFile serviceAccountPath - tokenResponse <- Auth.fetchToken manager serviceAccount scope - AuthT.tokenType tokenResponse `shouldBe` "Bearer" - AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600) + SystemEnv.lookupEnv "PUBSUB_EMULATOR_HOST" >>= \case + Just _ -> pendingWith "skipping as auth not supported in emulator" + Nothing -> do + SystemEnv.lookupEnv "GOOGLE_APPLICATION_CREDENTIALS" >>= \case + Just credPath -> do + -- Try ADC first + adcResult <- Aeson.eitherDecodeFileStrict credPath :: IO (Either String (Auth.ADCCredentials)) + case adcResult of + Right adcCreds | Auth.adcType adcCreds == Text.pack "authorized_user" -> do + adc <- Auth.readApplicationDefaultCredentialsFile credPath + tokenResponse <- Auth.fetchApplicationDefaultCredentialsToken manager adc scope + AuthT.tokenType tokenResponse `shouldBe` "Bearer" + AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600) + _ -> do + -- Try as ServiceAccount + serviceAccount <- Auth.readServiceAccountFile credPath + tokenResponse <- Auth.fetchToken manager serviceAccount scope + AuthT.tokenType tokenResponse `shouldBe` "Bearer" + AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600) + Nothing -> do + -- Try metadata server + tokenResponse <- Auth.fetchMetadataToken manager scope + AuthT.tokenType tokenResponse `shouldBe` "Bearer" + AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600) where scope = "https://www.googleapis.com/auth/pubsub" spec :: Spec diff --git a/test/Cloud/PubSub/TestHelpers.hs b/test/Cloud/PubSub/TestHelpers.hs index 0c5cf5d..0dfaa32 100644 --- a/test/Cloud/PubSub/TestHelpers.hs +++ b/test/Cloud/PubSub/TestHelpers.hs @@ -1,12 +1,16 @@ module Cloud.PubSub.TestHelpers where import qualified Cloud.PubSub as PubSub +import qualified Cloud.PubSub.Auth as Auth +import qualified Cloud.PubSub.Auth.Types as AuthT import Cloud.PubSub.Core.Types ( Message(..) , TopicName ) import qualified Cloud.PubSub.Core.Types as Core import qualified Cloud.PubSub.Http.Types as HttpT import qualified Cloud.PubSub.Logger as Logger +import qualified Data.Aeson as Aeson +import qualified Data.Text as Text import qualified Cloud.PubSub.Snapshot as Snapshot import qualified Cloud.PubSub.Snapshot.Types as SnapshotT import qualified Cloud.PubSub.Snapshot.Types as SnapshotT.NewSnapshot @@ -31,6 +35,7 @@ import qualified Data.HashMap.Strict as HM import Data.Text ( Text ) import qualified Data.Text as Text import Data.Time ( NominalDiffTime ) +import qualified System.Directory as SystemDir import qualified System.Environment as SystemEnv import qualified System.IO as SystemIO import Test.Hspec ( pendingWith ) @@ -151,37 +156,112 @@ getProjectId :: IO (Maybe Core.ProjectId) getProjectId = fmap (Core.ProjectId . Text.pack) <$> SystemEnv.lookupEnv "PROJECT_ID" +-- Helper to check if a file contains ADC credentials +-- Returns: Just True if ADC, Just False if ServiceAccount, Nothing if neither +detectCredentialType :: FilePath -> IO (Maybe Bool) +detectCredentialType filePath = do + -- First try to parse as ServiceAccount (has required fields that ADC doesn't) + saResult <- Aeson.eitherDecodeFileStrict filePath :: IO (Either String (AuthT.ServiceAccount)) + case saResult of + Right _ -> return $ Just False -- It's a ServiceAccount + Left _ -> do + -- Try to parse as ADC (authorized_user type) + adcResult <- Aeson.eitherDecodeFileStrict filePath :: IO (Either String (Auth.ADCCredentials)) + case adcResult of + Right (Auth.ADCCredentials t _ _ _) -> + if t == Text.pack "authorized_user" + then return $ Just True -- It's ADC with authorized_user + else return $ Just True -- It's ADC but different type (e.g. impersonated_service_account), treat as ADC + Left _ -> do + -- Try to read as raw JSON to check for type field + rawResult <- Aeson.eitherDecodeFileStrict filePath :: IO (Either String Aeson.Value) + case rawResult of + Right (Aeson.Object obj) -> do + -- Check if it has a "type" field that suggests it's some form of credentials + case Aeson.lookup "type" obj of + Just (Aeson.String credType) -> + if credType == Text.pack "service_account" + then return $ Just False -- It's a ServiceAccount (but parsing failed for some reason) + else return $ Just True -- It's some form of ADC or credential file + _ -> return Nothing -- No type field, can't determine + _ -> return Nothing -- Not an object, can't parse + getPubSubTarget :: NominalDiffTime -> IO PubSub.PubSubTarget getPubSubTarget renewThreshold = do maybeEmulatorHost <- SystemEnv.lookupEnv "PUBSUB_EMULATOR_HOST" - maybeSaFile <- SystemEnv.lookupEnv "GOOGLE_APPLICATION_CREDENTIALS" - case (maybeEmulatorHost, maybeSaFile) of - (Just hostAndPortStr, Nothing) -> + case maybeEmulatorHost of + Just hostAndPortStr -> return $ PubSub.EmulatorTarget $ PubSub.HostAndPort hostAndPortStr - (Nothing, Just saFile) -> - let authMethod = PubSub.ServiceAccountFile saFile - in return - $ PubSub.CloudServiceTarget - $ PubSub.CloudConfig renewThreshold authMethod - (_, _) -> - error - "Please specify either \"PUBSUB_EMULATOR_HOST\" or \ - \\"GOOGLE_APPLICATION_CREDENTIALS\" depending whether you to run \ - \the tests against the emulator or the cloud hosted version." + Nothing -> do + -- Check for GOOGLE_APPLICATION_CREDENTIALS (could be service account or ADC) + maybeCredFile <- SystemEnv.lookupEnv "GOOGLE_APPLICATION_CREDENTIALS" + case maybeCredFile of + Just credFile -> do + -- Try to detect if it's ADC or service account + credType <- detectCredentialType credFile + case credType of + Just True -> + -- It's ADC + let authMethod = PubSub.ApplicationDefaultCredentialsFile credFile + in return + $ PubSub.CloudServiceTarget + $ PubSub.CloudConfig renewThreshold authMethod + Just False -> + -- It's a ServiceAccount + let authMethod = PubSub.ServiceAccountFile credFile + in return + $ PubSub.CloudServiceTarget + $ PubSub.CloudConfig renewThreshold authMethod + Nothing -> do + -- Can't parse as either format, but file exists + -- This might be a WIF credentials file (e.g. impersonated_service_account type) + -- The google-github-actions/auth action creates credential files that Google's + -- client libraries can use but our custom parser doesn't support yet. + -- Since we can't parse it, we'll try to use it anyway - the error will be more + -- helpful if it fails. In practice, this should work with gcloud CLI or + -- Google's client libraries, but our custom implementation needs the file format. + -- For now, treat it as ADC and let readApplicationDefaultCredentialsFile handle the error. + let authMethod = PubSub.ApplicationDefaultCredentialsFile credFile + in return + $ PubSub.CloudServiceTarget + $ PubSub.CloudConfig renewThreshold authMethod + Nothing -> do + -- Try default ADC location if GOOGLE_APPLICATION_CREDENTIALS not set + maybeHome <- SystemEnv.lookupEnv "HOME" + case maybeHome of + Just homeDir -> do + let defaultADC = homeDir <> "/.config/gcloud/application_default_credentials.json" + adcExists <- SystemDir.doesFileExist defaultADC + if adcExists + then let authMethod = PubSub.ApplicationDefaultCredentialsFile defaultADC + in return + $ PubSub.CloudServiceTarget + $ PubSub.CloudConfig renewThreshold authMethod + else -- Fall back to metadata server (for GCP VMs) + let authMethod = PubSub.MetadataServer + in return + $ PubSub.CloudServiceTarget + $ PubSub.CloudConfig renewThreshold authMethod + Nothing -> -- Fall back to metadata server (for GCP VMs) + let authMethod = PubSub.MetadataServer + in return + $ PubSub.CloudServiceTarget + $ PubSub.CloudConfig renewThreshold authMethod usageMessage :: String usageMessage = unlines [ "Missing config: Tests can be run against the hosted Cloud PubSub service \ \or the emulator. The emulator does not have full API coverage and as such \ \some tests are not run when the tests are with the emulator." - , "To run tests with the hosted Cloud PubSub, please set the enviroment \ - \variable \"GOOGLE_APPLICATION_CREDENTIALS\" to the path to the \ - \service account keys in JSON format and specify Google Cloud Project ID \ - \via the \"PROJECT_ID\" environment variable. Given that service accounts \ - \can be used across projects \"project_id\" field in the JSON key file \ - \is ignored." - , "To run against the emulator please start the emulator and set the \ - \\"PUBSUB_EMULATOR_HOST\" and the \"PROJECT_ID\" environment variables." + , "To run tests with the hosted Cloud PubSub:" + , " Option 1 (recommended for local): Run 'gcloud auth application-default login' \ + \then set \"PROJECT_ID\" environment variable" + , " Option 2 (legacy): Set \"GOOGLE_APPLICATION_CREDENTIALS\" to the path \ + \to service account keys in JSON format and \"PROJECT_ID\"" + , " Option 3 (GCP VM/CI): Use metadata server authentication (default when \ + \GOOGLE_APPLICATION_CREDENTIALS is not set and ADC file doesn't exist)" + , "To run against the emulator: Start the emulator and set \"PUBSUB_EMULATOR_HOST\" \ + \and \"PROJECT_ID\" environment variables." ] mkTestPubSubEnvWithRenewThreshold :: NominalDiffTime -> IO TestEnv