Switch to Workload Identity Federation and Add Application Default Credentials Support

.github/workflows/stack-cloud.yaml

@@ -4,15 +4,31 @@ jobs:
   build-and-test:
     name: Build and Test
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v2
       - uses: haskell-actions/setup@v2
         with:
           enable-stack: true
           stack-no-global: true
           stack-version: "3.3.1"
+      - id: auth
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v3
+        with:
+          workload_identity_provider: projects/3367369000/locations/global/workloadIdentityPools/wip-github-yqou/providers/wip-provider-github
+          service_account: sa-ci-cloud-pubsub@proda-ci.iam.gserviceaccount.com
+          create_credentials_file: true
+          export_environment_variables: true
+          universe: googleapis.com
+          cleanup_credentials: true
+          access_token_lifetime: 3600s
+          access_token_scopes: https://www.googleapis.com/auth/cloud-platform
+          id_token_include_email: false
       - name: Cache Stack Dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             ~/.stack
@@ -23,12 +39,7 @@ jobs:
             - ${{ runner.os }}-
       - name: Install Dependencies
         run: stack install --only-dependencies --test
-      - run: 'mkdir secrets && echo "$GCP_SA_KEY" > ./secrets/service_account.json'
-        shell: bash
-        env:
-          GCP_SA_KEY: ${{secrets.GCP_SA_KEY}}
       - name: Run Tests
         run: stack test
         env:
           PROJECT_ID: ${{secrets.PROJECT_ID}}
-          GOOGLE_APPLICATION_CREDENTIALS: ./secrets/service_account.json

package.yaml

@@ -115,6 +115,7 @@ tests:
       - async
       - bytestring
       - cloud-pubsub
+      - directory
       - exceptions
       - hspec
       # See https://github.com/input-output-hk/haskell.nix/issues/231

set-env.sh

@@ -10,8 +10,12 @@ if [[ -z "$PUBSUB_EMULATOR_HOST" ]] && [[ -z "$GOOGLE_APPLICATION_CREDENTIALS" ]
     PUBSUB_EMULATOR_HOST="localhost:8085"
 fi
 
-# To run tests against hosted Cloud PubSub set GOOGLE_APPLICATION_CREDENTIALS
-# instead of PUBSUB_EMULATOR_HOST
-# GOOGLE_APPLICATION_CREDENTIALS="secrets/service_account.json"
+# To run tests against hosted Cloud PubSub:
+# Option 1 (recommended for local): Use gcloud auth application-default login
+#   This creates credentials at ~/.config/gcloud/application_default_credentials.json
+#   No need to set GOOGLE_APPLICATION_CREDENTIALS
+# Option 2 (legacy): Set GOOGLE_APPLICATION_CREDENTIALS to a service account key file
+#   GOOGLE_APPLICATION_CREDENTIALS="secrets/service_account.json"
+# Option 3 (GCP VM/CI): Don't set GOOGLE_APPLICATION_CREDENTIALS, use metadata server
 
 set +o allexport

src/Cloud/PubSub.hs

@@ -24,6 +24,7 @@ import qualified Network.HTTP.Client.TLS       as HttpClientTLS
 
 data AuthMethod = ServiceAccountFile FilePath
                 | MetadataServer
+                | ApplicationDefaultCredentialsFile FilePath
   deriving (Show, Eq)
 
 newtype HostAndPort = HostAndPort { unwrapHostAndPort :: String }
@@ -53,6 +54,8 @@ mkClientResources projectId target = do
               AuthT.FromServiceAccount <$> Auth.readServiceAccountFile saFile
             MetadataServer ->
               pure AuthT.FromMetadataServer
+            ApplicationDefaultCredentialsFile adcFile ->
+              AuthT.FromApplicationDefaultCredentials <$> Auth.readApplicationDefaultCredentialsFile adcFile
           let resources  = CloudTargetResources authSource tokenVar threshold
               serviceUrl = "https://pubsub.googleapis.com"
           return (serviceUrl, Cloud resources)

src/Cloud/PubSub/Auth.hs

@@ -2,6 +2,9 @@
   ( fetchToken
   , readServiceAccountFile
   , fetchMetadataToken
+  , readApplicationDefaultCredentialsFile
+  , fetchApplicationDefaultCredentialsToken
+  , ADCCredentials(..)
   ) where
 
 import qualified Cloud.PubSub.Auth.Types       as AuthT
@@ -11,10 +14,12 @@
 import qualified Crypto.PubKey.RSA.PKCS15      as RSA
 import qualified Data.Aeson                    as Aeson
 import           Data.Aeson                     ( (.=) )
+import qualified Data.HashMap.Strict           as HashMap
 import           Data.ByteString                ( ByteString )
 import qualified Data.ByteString.Base64.URL    as Base64
 import qualified Data.ByteString.Lazy          as LBS
 import           Data.Functor                   ( (<&>) )
+import           GHC.Generics                   ( Generic )
 import qualified Data.Text                     as Text
 import qualified Data.Text.Encoding            as TE
 import qualified Data.Time                     as Time
@@ -25,6 +30,48 @@
 readServiceAccountFile fp =
   liftIO $ Aeson.eitherDecodeFileStrict fp >>= either fail return
 
+data ADCCredentials = ADCCredentials
+  { adcType         :: Text.Text
+  , adcClientId     :: Text.Text
+  , adcClientSecret :: Text.Text
+  , adcRefreshToken :: Text.Text
+  }
+  deriving stock (Generic, Show)
+
+instance Aeson.FromJSON ADCCredentials where
+  parseJSON = Aeson.genericParseJSON $ Aeson.defaultOptions
+    { Aeson.fieldLabelModifier = Aeson.camelTo2 '_' . drop 3
+    }
+
+readApplicationDefaultCredentialsFile :: MonadIO m => FilePath -> m AuthT.ApplicationDefaultCredentials
+readApplicationDefaultCredentialsFile fp = liftIO $ do
+  result <- Aeson.eitherDecodeFileStrict fp :: IO (Either String ADCCredentials)
+  case result of
+    Right adc | adcType adc == Text.pack "authorized_user" ->
+      return $ AuthT.ApplicationDefaultCredentials
+        { AuthT.adcClientId     = adcClientId adc
+        , AuthT.adcClientSecret = adcClientSecret adc
+        , AuthT.adcRefreshToken = adcRefreshToken adc
+        }
+    Right adc ->
+      fail $ "Unsupported ADC credentials type: " <> Text.unpack (adcType adc) <>
+             ". Only 'authorized_user' type is supported. For WIF/impersonated service accounts," <>
+             " please use environment variables or a supported credential format."
+    Left err -> do
+      -- Try to read as raw JSON to provide better error message
+      rawResult <- Aeson.eitherDecodeFileStrict fp :: IO (Either String Aeson.Value)
+      case rawResult of
+        Right (Aeson.Object obj) ->
+          case HashMap.lookup (Text.pack "type") obj of
+            Just (Aeson.String credType) ->
+              fail $ "Unsupported credential type: " <> Text.unpack credType <>
+                     ". Expected 'authorized_user' ADC format with client_id, client_secret, and refresh_token fields."
+            _ ->
+              fail $ "Could not parse credentials file. Expected ADC format with 'authorized_user' type. " <>
+                     "Parse error: " <> err
+        _ ->
+          fail $ "Could not parse credentials file as JSON. Parse error: " <> err
+
 createAssertionTokenBody
   :: AuthT.PrivateKeyId -> AuthT.TokenClaims -> ByteString
 createAssertionTokenBody (AuthT.PrivateKeyId keyId) tokenClaims =
@@ -107,3 +154,29 @@
           )
   response <- HTTP.httpJSON request
   return $ HTTP.getResponseBody response
+
+-- | Fetches a GCP access token using Application Default Credentials (from gcloud auth application-default login)
+-- Implements the OAuth2 refresh token flow described here:
+-- https://developers.google.com/identity/protocols/oauth2/web-server#offline
+fetchApplicationDefaultCredentialsToken
+  :: MonadIO m
+  => HttpClientC.Manager
+  -> AuthT.ApplicationDefaultCredentials
+  -> AuthT.Scope
+  -> m AuthT.AccessTokenResponse
+fetchApplicationDefaultCredentialsToken manager adc scope = liftIO $ do
+  let tokenUrl = "https://oauth2.googleapis.com/token"
+      formData =
+        [ ("client_id"    , TE.encodeUtf8 $ AuthT.adcClientId adc)
+        , ("client_secret", TE.encodeUtf8 $ AuthT.adcClientSecret adc)
+        , ("refresh_token" , TE.encodeUtf8 $ AuthT.adcRefreshToken adc)
+        , ("grant_type"   , "refresh_token")
+        , ("scope"        , TE.encodeUtf8 $ AuthT.unwrapScope scope)
+        ]
+  request <-
+    HTTP.parseRequest ("POST " <> tokenUrl)
+      <&> ( HTTP.setRequestBodyURLEncoded formData
+          . HTTP.setRequestManager manager
+          )
+  response <- HTTP.httpJSON request
+  return $ HTTP.getResponseBody response

src/Cloud/PubSub/Auth/Token.hs

@@ -28,6 +28,8 @@ fetchAndUpdateToken resources manager scope = do
       Auth.fetchToken manager serviceAccount scope
     Auth.FromMetadataServer ->
       Auth.fetchMetadataToken manager scope
+    Auth.FromApplicationDefaultCredentials adc ->
+      Auth.fetchApplicationDefaultCredentialsToken manager adc scope
   now           <- Time.getCurrentTime
   let accessToken = tokenResponse.accessToken
       expiresAt   = Time.addUTCTime tokenResponse.expiresIn now

src/Cloud/PubSub/Auth/Types.hs

@@ -7,6 +7,7 @@ module Cloud.PubSub.Auth.Types
   , Scope(..)
   , ServiceAccount(..)
   , TokenSource(..)
+  , ApplicationDefaultCredentials(..)
   , TokenClaims(..)
   , UnixEpochSeconds(..)
   , X509PrivateKey(..)
@@ -62,8 +63,15 @@ instance Aeson.FromJSON ServiceAccount where
     { Aeson.fieldLabelModifier = Aeson.camelTo2 '_' . drop 2
     }
 
+data ApplicationDefaultCredentials = ApplicationDefaultCredentials
+  { adcClientId     :: Text
+  , adcClientSecret :: Text
+  , adcRefreshToken :: Text
+  }
+
 data TokenSource = FromServiceAccount ServiceAccount
                  | FromMetadataServer
+                 | FromApplicationDefaultCredentials ApplicationDefaultCredentials
 
 newtype AccessToken = AccessToken
   { unwrapAccessToken :: Text

test/Cloud/PubSub/AuthSpec.hs

@@ -2,6 +2,8 @@ module Cloud.PubSub.AuthSpec where
 
 import qualified Cloud.PubSub.Auth             as Auth
 import qualified Cloud.PubSub.Auth.Types       as AuthT
+import qualified Data.Aeson                   as Aeson
+import qualified Data.Text                     as Text
 import qualified Network.HTTP.Client           as HttpClient
 import qualified System.Environment            as SystemEnv
 import qualified Network.HTTP.Client.TLS       as HttpClientTLS
@@ -10,13 +12,30 @@ import           Test.Hspec
 tokenGetTest :: IO ()
 tokenGetTest = do
   manager        <- HttpClient.newManager HttpClientTLS.tlsManagerSettings
-  SystemEnv.lookupEnv "GOOGLE_APPLICATION_CREDENTIALS" >>= \case
-    Nothing -> pendingWith "skipping as auth not supported in emulator"
-    Just serviceAccountPath -> do
-      serviceAccount <- Auth.readServiceAccountFile serviceAccountPath
-      tokenResponse  <- Auth.fetchToken manager serviceAccount scope
-      AuthT.tokenType tokenResponse `shouldBe` "Bearer"
-      AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600)
+  SystemEnv.lookupEnv "PUBSUB_EMULATOR_HOST" >>= \case
+    Just _ -> pendingWith "skipping as auth not supported in emulator"
+    Nothing -> do
+      SystemEnv.lookupEnv "GOOGLE_APPLICATION_CREDENTIALS" >>= \case
+        Just credPath -> do
+          -- Try ADC first
+          adcResult <- Aeson.eitherDecodeFileStrict credPath :: IO (Either String (Auth.ADCCredentials))
+          case adcResult of
+            Right adcCreds | Auth.adcType adcCreds == Text.pack "authorized_user" -> do
+              adc <- Auth.readApplicationDefaultCredentialsFile credPath
+              tokenResponse <- Auth.fetchApplicationDefaultCredentialsToken manager adc scope
+              AuthT.tokenType tokenResponse `shouldBe` "Bearer"
+              AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600)
+            _ -> do
+              -- Try as ServiceAccount
+              serviceAccount <- Auth.readServiceAccountFile credPath
+              tokenResponse  <- Auth.fetchToken manager serviceAccount scope
+              AuthT.tokenType tokenResponse `shouldBe` "Bearer"
+              AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600)
+        Nothing -> do
+          -- Try metadata server
+          tokenResponse <- Auth.fetchMetadataToken manager scope
+          AuthT.tokenType tokenResponse `shouldBe` "Bearer"
+          AuthT.expiresIn tokenResponse `shouldSatisfy` (\t -> 3595 < t && t < 3600)
       where scope = "https://www.googleapis.com/auth/pubsub"
 
 spec :: Spec