Skip to content

Commit 75163cb

Browse files
stevenc81stevenc81
committed
use a seperate service account with more restrictive permission levels
1 parent 7233f80 commit 75163cb

3 files changed

Lines changed: 149 additions & 4 deletions

File tree

‎tools/cloud_functions/bq_table_snapshots/terraform/function.tf‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ resource "google_cloudfunctions_function" "bq_backup_fetch_tables_names" {
9191
entry_point = "main"
9292
source_archive_bucket = google_storage_bucket.bucket.name
9393
source_archive_object = google_storage_bucket_object.bq_backup_fetch_tables_names.name
94+
service_account_email = google_service_account.fetcher.email
9495

9596
environment_variables = {
9697
DATA_PROJECT_ID = var.storage_project_id
@@ -102,6 +103,11 @@ resource "google_cloudfunctions_function" "bq_backup_fetch_tables_names" {
102103
event_type = "providers/cloud.pubsub/eventTypes/topic.publish"
103104
resource = google_pubsub_topic.snapshot_dataset_topic.id
104105
}
106+
107+
depends_on = [
108+
google_bigquery_dataset_iam_member.fetcher_source_dataset,
109+
google_pubsub_topic_iam_member.fetcher_pubsub
110+
]
105111
}
106112

107113
##########################################
@@ -128,6 +134,7 @@ resource "google_cloudfunctions_function" "bq_backup_create_snapshots" {
128134
entry_point = "main"
129135
source_archive_bucket = google_storage_bucket.bucket.name
130136
source_archive_object = google_storage_bucket_object.bq_backup_create_snapshots.name
137+
service_account_email = google_service_account.creator.email
131138

132139
environment_variables = {
133140
BQ_DATA_PROJECT_ID = var.storage_project_id
@@ -138,5 +145,11 @@ resource "google_cloudfunctions_function" "bq_backup_create_snapshots" {
138145
event_type = "providers/cloud.pubsub/eventTypes/topic.publish"
139146
resource = google_pubsub_topic.bq_snapshot_create_snapshot_topic.id
140147
}
148+
149+
depends_on = [
150+
google_project_iam_member.creator_job_user,
151+
google_bigquery_dataset_iam_member.creator_source_dataset,
152+
google_bigquery_dataset_iam_member.creator_target_dataset
153+
]
141154
}
142155

‎tools/cloud_functions/bq_table_snapshots/terraform/iam.tf‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Service Accounts for BigQuery Snapshot Functions
2+
#
3+
# This configuration implements least privilege access by creating dedicated
4+
# service accounts with minimal permissions for each Cloud Function.
5+
#
6+
# Architecture:
7+
# - sa-bq-snap-fetcher: Lists tables in source dataset, publishes to Pub/Sub
8+
# - sa-bq-snap-creator: Creates snapshots from source to target dataset
9+
10+
# =============================================================================
11+
# Service Accounts
12+
# =============================================================================
13+
14+
resource "google_service_account" "fetcher" {
15+
account_id = "sa-bq-snap-fetcher"
16+
display_name = "BigQuery Snapshot Fetcher Service Account"
17+
description = "Service account for listing tables in source dataset and triggering snapshot creation"
18+
project = var.project_id
19+
}
20+
21+
resource "google_service_account" "creator" {
22+
account_id = "sa-bq-snap-creator"
23+
display_name = "BigQuery Snapshot Creator Service Account"
24+
description = "Service account for creating BigQuery table snapshots from source to target dataset"
25+
project = var.project_id
26+
}
27+
28+
# =============================================================================
29+
# Custom IAM Roles
30+
# =============================================================================
31+
32+
# Custom role for fetcher function
33+
# Permissions: list tables and get dataset metadata from source dataset
34+
resource "google_project_iam_custom_role" "bq_snapshot_fetcher" {
35+
project = var.project_id
36+
role_id = "bqSnapshotFetcher"
37+
title = "BigQuery Snapshot Fetcher"
38+
description = "Minimal permissions to list tables in a dataset for snapshot processing"
39+
permissions = [
40+
"bigquery.tables.list",
41+
"bigquery.datasets.get"
42+
]
43+
}
44+
45+
# NOTE: Using predefined BigQuery roles instead of custom roles.
46+
# Per Google Cloud documentation, only specific predefined roles (dataOwner, admin,
47+
# studioAdmin) can create snapshots with expiration times. Custom roles cannot work
48+
# for this use case due to BigQuery API limitations.
49+
50+
# =============================================================================
51+
# IAM Bindings - Fetcher Service Account
52+
# =============================================================================
53+
54+
# Grant fetcher SA permissions to list tables in source dataset
55+
resource "google_bigquery_dataset_iam_member" "fetcher_source_dataset" {
56+
project = var.storage_project_id
57+
dataset_id = var.source_dataset_name
58+
role = google_project_iam_custom_role.bq_snapshot_fetcher.id
59+
member = "serviceAccount:${google_service_account.fetcher.email}"
60+
}
61+
62+
# Grant fetcher SA permissions to publish messages to snapshot trigger topic
63+
resource "google_pubsub_topic_iam_member" "fetcher_pubsub" {
64+
project = var.project_id
65+
topic = google_pubsub_topic.bq_snapshot_create_snapshot_topic.name
66+
role = "roles/pubsub.publisher"
67+
member = "serviceAccount:${google_service_account.fetcher.email}"
68+
}
69+
70+
# =============================================================================
71+
# IAM Bindings - Creator Service Account
72+
# =============================================================================
73+
74+
# Grant creator SA read permissions on source dataset
75+
# Using predefined dataViewer role which includes:
76+
# - bigquery.tables.get (read metadata)
77+
# - bigquery.tables.getData (read data for time-travel)
78+
# - bigquery.tables.createSnapshot (create snapshot from source)
79+
# - bigquery.datasets.get (access dataset)
80+
resource "google_bigquery_dataset_iam_member" "creator_source_dataset" {
81+
project = var.storage_project_id
82+
dataset_id = var.source_dataset_name
83+
role = "roles/bigquery.dataViewer"
84+
member = "serviceAccount:${google_service_account.creator.email}"
85+
}
86+
87+
# Grant creator SA write permissions on target dataset
88+
# Using predefined dataOwner role which includes:
89+
# - bigquery.tables.create (create snapshot tables)
90+
# - bigquery.tables.createSnapshot (snapshot operation)
91+
# - bigquery.tables.deleteSnapshot (REQUIRED for expiration)
92+
# - bigquery.tables.updateData (write snapshot data)
93+
# - bigquery.tables.update (update table metadata)
94+
# - bigquery.tables.delete (cleanup old snapshots)
95+
# - bigquery.datasets.get (access dataset)
96+
# - bigquery.tables.setIamPolicy (manage table permissions)
97+
#
98+
# IMPORTANT: Per Google Cloud documentation, ONLY bigquery.dataOwner,
99+
# bigquery.admin, and bigquery.studioAdmin can create snapshots with
100+
# expiration times. This is a BigQuery API limitation - dataEditor lacks
101+
# bigquery.tables.deleteSnapshot which is required for setting expiration.
102+
# dataOwner is the least privileged role that supports snapshot expiration.
103+
resource "google_bigquery_dataset_iam_member" "creator_target_dataset" {
104+
project = var.storage_project_id
105+
dataset_id = google_bigquery_dataset.dataset.dataset_id
106+
role = "roles/bigquery.dataOwner"
107+
member = "serviceAccount:${google_service_account.creator.email}"
108+
}
109+
110+
# Grant creator SA permissions to create BigQuery jobs at project level
111+
# Note: BigQuery jobs are project-scoped resources, so this must be project-level.
112+
# This is a known limitation - the SA can create any BigQuery job type, but this
113+
# is the minimum required for snapshot operations.
114+
resource "google_project_iam_member" "creator_job_user" {
115+
project = var.project_id
116+
role = "roles/bigquery.jobUser"
117+
member = "serviceAccount:${google_service_account.creator.email}"
118+
}
119+
120+
# =============================================================================
121+
# Outputs
122+
# =============================================================================
123+
124+
output "fetcher_service_account_email" {
125+
description = "Email of the fetcher service account"
126+
value = google_service_account.fetcher.email
127+
}
128+
129+
output "creator_service_account_email" {
130+
description = "Email of the creator service account"
131+
value = google_service_account.creator.email
132+
}

‎tools/cloud_functions/bq_table_snapshots/terraform/terraform.tfvars‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
project_id = "buffer-data"
22
storage_project_id = "buffer-data"
3-
source_dataset_name = "dbt_steven"
4-
target_dataset_name = "dbt_steven_snapshots"
5-
crontab_format = "15 * * * *"
6-
seconds_before_expiration = 604800
3+
source_dataset_name = "dbt_buffer"
4+
target_dataset_name = "dbt_buffer_snapshots"
5+
crontab_format = "0 2 * * *"
6+
seconds_before_expiration = 2419200

0 commit comments

Comments
 (0)