diff --git a/.gitignore b/.gitignore index 553275c..f0174c7 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ *.tfvars *.pem vars +.helmignore diff --git a/README.md b/README.md index 49cc740..9038012 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,12 @@ module "rds-pg" { name = "postgresql" db_name = "proddb" vpc_id = "vpc-047eb8acfb73" - multi_az = "true" + multi_az = false subnet_ids = ["subnet-b39cfc", "subnet-090b8d8"] environment = "prod" + create_namespace = true storage_type = "gp3" + cluster_name = "" replica_enable = false replica_count = 1 kms_key_arn = "arn:aws:kms:region:2222222222:key/f8c8d802-a34b" @@ -54,6 +56,20 @@ module "rds-pg" { slack_channel = "skaf-dev" slack_webhook_url = "https://hooks/xxxxxxxx" custom_user_password = "postgresqlpasswd" + cluster_name = "" + namespace = local.namespace + create_namespace = local.create_namespace + postgresdb_backup_enabled = false + postgresdb_backup_config = { + postgres_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases + cron_for_full_backup = "*/2 * * * *" # set cronjob for backup + bucket_uri = "s3://mongodb-backups-atmosly" # s3 bucket uri + } + postgresdb_restore_enabled = false + postgresdb_restore_config = { + bucket_uri = "s3://mongodb-backups-atmosly" #S3 bucket URI (without a trailing slash /) containing the backup dump file. + backup_file_name = "db5_20241114111607.sql" #Give .sql or .zip file for restore + } } ``` Refer [examples](https://github.com/squareops/terraform-aws-rds-postgresql/tree/main/examples) for more details. @@ -66,51 +82,52 @@ The required IAM permissions to create resources from this module can be found [ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 0.13 | -| [aws](#requirement\_aws) | 5.13.1 | +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | >= 5.0.0 | ## Providers | Name | Version | |------|---------| | [archive](#provider\_archive) | n/a | -| [aws](#provider\_aws) | 5.13.1 | +| [aws](#provider\_aws) | >= 5.0.0 | | [random](#provider\_random) | n/a | ## Modules | Name | Source | Version | |------|--------|---------| +| [backup\_restore](#module\_backup\_restore) | ./modules/db-backup-restore | n/a | | [cw\_sns\_slack](#module\_cw\_sns\_slack) | ./lambda | n/a | | [db](#module\_db) | terraform-aws-modules/rds/aws | 6.1.0 | | [db\_replica](#module\_db\_replica) | terraform-aws-modules/rds/aws | 6.1.0 | -| [security\_group\_rds](#module\_security\_group\_rds) | terraform-aws-modules/security-group/aws | ~> 4 | +| [security\_group\_rds](#module\_security\_group\_rds) | terraform-aws-modules/security-group/aws | ~> 5.0 | ## Resources | Name | Type | |------|------| -| [aws_cloudwatch_metric_alarm.cache_cpu](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/cloudwatch_metric_alarm) | resource | -| [aws_cloudwatch_metric_alarm.disk_free_storage_space_too_low](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/cloudwatch_metric_alarm) | resource | -| [aws_kms_ciphertext.slack_url](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/kms_ciphertext) | resource | -| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/kms_key) | resource | -| [aws_lambda_permission.sns_lambda_slack_invoke](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/lambda_permission) | resource | -| [aws_secretsmanager_secret.secret_master_db](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/secretsmanager_secret) | resource | -| [aws_secretsmanager_secret_version.rds_credentials](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/secretsmanager_secret_version) | resource | -| [aws_security_group_rule.cidr_ingress](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.default_ingress](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/security_group_rule) | resource | -| [aws_sns_topic.slack_topic](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/sns_topic) | resource | -| [aws_sns_topic_subscription.slack-endpoint](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/resources/sns_topic_subscription) | resource | +| [aws_cloudwatch_metric_alarm.cache_cpu](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource | +| [aws_cloudwatch_metric_alarm.disk_free_storage_space_too_low](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource | +| [aws_kms_ciphertext.slack_url](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_ciphertext) | resource | +| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_lambda_permission.sns_lambda_slack_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_secretsmanager_secret.secret_master_db](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | +| [aws_secretsmanager_secret_version.rds_credentials](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | +| [aws_security_group_rule.cidr_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.default_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_sns_topic.slack_topic](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource | +| [aws_sns_topic_subscription.slack-endpoint](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource | | [random_password.master](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [archive_file.lambdazip](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/data-sources/availability_zones) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/5.13.1/docs/data-sources/region) | data source | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [additional\_tags](#input\_additional\_tags) | A map of additional tags to apply to the AWS resources | `map(string)` | 
{
  "automation": "true"
}
| no | +| [additional\_tags](#input\_additional\_tags) | A map of additional tags to apply to the AWS resources | `map(string)` | 
{
  "automation": "true"
}
| no | | [alarm\_actions](#input\_alarm\_actions) | Alarm action list | `list(string)` | `[]` | no | | [alarm\_cpu\_threshold\_percent](#input\_alarm\_cpu\_threshold\_percent) | CPU threshold alarm level | `number` | `75` | no | | [allocated\_storage](#input\_allocated\_storage) | The allocated storage capacity for the database in gibibytes (GiB) | `number` | `20` | no | @@ -119,8 +136,11 @@ The required IAM permissions to create resources from this module can be found [ | [apply\_immediately](#input\_apply\_immediately) | Specifies whether any cluster modifications are applied immediately or during the next maintenance window | `bool` | `false` | no | | [backup\_retention\_period](#input\_backup\_retention\_period) | The number of days to retain backups for | `number` | `5` | no | | [backup\_window](#input\_backup\_window) | The preferred window for taking automated backups of the database | `string` | `"03:00-06:00"` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | | [cloudwatch\_metric\_alarms\_enabled](#input\_cloudwatch\_metric\_alarms\_enabled) | Boolean flag to enable/disable CloudWatch metrics alarms | `bool` | `false` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | | [create\_db\_subnet\_group](#input\_create\_db\_subnet\_group) | Whether to create a database subnet group | `bool` | `true` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | | [create\_security\_group](#input\_create\_security\_group) | Whether to create a security group for the database | `bool` | `true` | no | | [custom\_user\_password](#input\_custom\_user\_password) | Custom password for the RDS master user | `string` | `""` | no | | [cw\_sns\_topic\_arn](#input\_cw\_sns\_topic\_arn) | The username to use when sending notifications to Slack. | `string` | `""` | no | @@ -143,10 +163,15 @@ The required IAM permissions to create resources from this module can be found [ | [max\_allocated\_storage](#input\_max\_allocated\_storage) | The Maximum storage capacity for the database value after autoscaling | `number` | `null` | no | | [multi\_az](#input\_multi\_az) | Enable multi-AZ for disaster recovery | `bool` | `false` | no | | [name](#input\_name) | The name of the RDS instance | `string` | `""` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"postgresdb"` | no | | [ok\_actions](#input\_ok\_actions) | The list of actions to execute when this alarm transitions into an OK state from any other state. Each action is specified as an Amazon Resource Number (ARN) | `list(string)` | `[]` | no | | [performance\_insights\_enabled](#input\_performance\_insights\_enabled) | Specifies whether Performance Insights are enabled | `bool` | `false` | no | | [performance\_insights\_retention\_period](#input\_performance\_insights\_retention\_period) | The amount of time in days to retain Performance Insights data. Valid values are `7`, `731` (2 years) or a multiple of `31` | `number` | `7` | no | | [port](#input\_port) | The port number for the database | `number` | `5432` | no | +| [postgresdb\_backup\_config](#input\_postgresdb\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` | 
{
  "bucket_uri": "",
  "cron_for_full_backup": "",
  "postgres_database_name": ""
}
| no | +| [postgresdb\_backup\_enabled](#input\_postgresdb\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [postgresdb\_restore\_config](#input\_postgresdb\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` | 
{
  "bucket_uri": "",
  "file_name": ""
}
| no | +| [postgresdb\_restore\_enabled](#input\_postgresdb\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | | [publicly\_accessible](#input\_publicly\_accessible) | Specifies whether the RDS instance is publicly accessible over the internet | `bool` | `false` | no | | [random\_password\_length](#input\_random\_password\_length) | The length of the randomly generated password for the RDS primary cluster (default: 16) | `number` | `16` | no | | [replica\_count](#input\_replica\_count) | The number of replica instance | `number` | `1` | no | @@ -171,6 +196,7 @@ The required IAM permissions to create resources from this module can be found [ | [db\_instance\_name](#output\_db\_instance\_name) | Name of the database instance | | [db\_instance\_password](#output\_db\_instance\_password) | Password for accessing the database. | | [db\_instance\_username](#output\_db\_instance\_username) | Master username for accessing the database. | +| [db\_name](#output\_db\_name) | The database name used in the RDS module | | [db\_parameter\_group\_id](#output\_db\_parameter\_group\_id) | ID of the parameter group associated with the RDS instance. | | [db\_subnet\_group\_id](#output\_db\_subnet\_group\_id) | ID of the subnet group associated with the RDS instance. | | [master\_credential\_secret\_arn](#output\_master\_credential\_secret\_arn) | The ARN of the master user secret (Only available when manage\_master\_user\_password is set to true) | diff --git a/examples/complete-psql-replica/main.tf b/examples/complete-psql-replica/main.tf index 8b74987..a0988b3 100644 --- a/examples/complete-psql-replica/main.tf +++ b/examples/complete-psql-replica/main.tf @@ -4,6 +4,8 @@ locals { family = "postgres15" vpc_cidr = "10.20.0.0/16" environment = "prod" + create_namespace = true + namespace = "postgres" storage_type = "gp3" engine_version = "15.2" instance_class = "db.m5d.large" @@ -113,4 +115,23 @@ module "rds-pg" { slack_channel = "postgresql-notification" slack_webhook_url = "https://hooks/xxxxxxxx" custom_user_password = local.custom_user_password + #if you want backup and restore then you have to create your cluster with rds vpc , subnet, key_arn. + #And allow cluster security group in rds security group + # cluster_name = "cluster-name" + # namespace = local.namespace + # create_namespace = local.create_namespace + # postgresdb_backup_enabled = false + # postgresdb_backup_config = { + # postgres_database_name = "" # which database backup you want + # s3_bucket_region = "" #s3 bucket region + # cron_for_full_backup = "*/3 * * * *" + # bucket_uri = "s3://xyz" #s3 bucket uri + # } + # postgresdb_restore_enabled = false + # postgresdb_restore_config = { + # bucket_uri = "s3://xyz" #s3 bucket uri which have dackup dump file + # backup_file_name = "abc.dump" #Give only .sql or .zip file for restore + # s3_bucket_region = "" # bucket region + # DB_NAME = "" # which db to restore backup file + # } } diff --git a/examples/complete/README.md b/examples/complete/README.md index d056d1b..9b26e16 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -14,20 +14,20 @@ This example will be very useful for users who are new to a module and want to q | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0 | -| [aws](#requirement\_aws) | >= 3.43.0 | +| [aws](#requirement\_aws) | >= 5.0.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.43.0 | +| [aws](#provider\_aws) | >= 5.0.0 | ## Modules | Name | Source | Version | |------|--------|---------| -| [kms](#module\_kms) | terraform-aws-modules/kms/aws | n/a | -| [rds-pg](#module\_rds-pg) | ../../ | n/a | +| [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 1.0 | +| [rds-pg](#module\_rds-pg) | squareops/rds-postgresql/aws | 2.0.0 | | [vpc](#module\_vpc) | squareops/vpc/aws | n/a | ## Resources @@ -35,6 +35,8 @@ This example will be very useful for users who are new to a module and want to q | Name | Type | |------|------| | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | +| [aws_eks_cluster_auth.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | ## Inputs @@ -45,6 +47,7 @@ No inputs. | Name | Description | |------|-------------| +| [db\_name](#output\_db\_name) | Database name | | [instance\_endpoint](#output\_instance\_endpoint) | Connection endpoint of the RDS instance. | | [instance\_name](#output\_instance\_name) | Name of the database instance. | | [instance\_password](#output\_instance\_password) | Password for accessing the database (Note: Terraform does not track this password after initial creation). | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 3223e4c..6a5c226 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -1,14 +1,19 @@ locals { - region = "us-east-2" + region = "us-east-1" name = "postgresql" family = "postgres15" vpc_cidr = "10.20.0.0/16" environment = "prod" - engine_version = "15.2" - instance_class = "db.m5d.large" + create_namespace = true + namespace = "pg" + engine_version = "15.7" + instance_class = "db.t4g.micro" storage_type = "gp3" + cluster_name = "" + replica_count = 1 + replica_enable = false current_identity = data.aws_caller_identity.current.arn - allowed_security_groups = ["sg-0a680afd35"] + allowed_security_groups = ["sg-xxxxxxxxxxxxxx"] custom_user_password = "" additional_tags = { Owner = "Organization_Name" @@ -21,8 +26,8 @@ data "aws_caller_identity" "current" {} data "aws_region" "current" {} module "kms" { - source = "terraform-aws-modules/kms/aws" - + source = "terraform-aws-modules/kms/aws" + version = "~> 1.0" deletion_window_in_days = 7 description = "Complete key example showing various configurations available" enable_key_rotation = true @@ -98,12 +103,14 @@ module "vpc" { module "rds-pg" { source = "squareops/rds-postgresql/aws" + version = "2.0.0" name = local.name - db_name = "postgres" - multi_az = "true" + db_name = "test" + multi_az = false family = local.family vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.database_subnets ## db subnets + allowed_security_groups = local.allowed_security_groups + subnet_ids = module.vpc.database_subnets environment = local.environment kms_key_arn = module.kms.key_arn storage_type = local.storage_type @@ -119,7 +126,7 @@ module "rds-pg" { final_snapshot_identifier_prefix = "final" major_engine_version = local.engine_version deletion_protection = false - cloudwatch_metric_alarms_enabled = true + cloudwatch_metric_alarms_enabled = false alarm_cpu_threshold_percent = 70 disk_free_storage_space = "10000000" # in bytes slack_notification_enabled = false @@ -127,4 +134,20 @@ module "rds-pg" { slack_channel = "postgresql-notification" slack_webhook_url = "https://hooks/xxxxxxxx" custom_user_password = local.custom_user_password + #if you want backup and restore then you have to create your cluster with rds vpc id , private subnets, kms key. + #And allow cluster security group in rds security group + cluster_name = local.cluster_name + namespace = local.namespace + create_namespace = local.create_namespace + postgresdb_backup_enabled = false + postgresdb_backup_config = { + postgres_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases + cron_for_full_backup = "*/2 * * * *" # set cronjob for backup + bucket_uri = "s3://my-backup-dumps-databases" # s3 bucket uri + } + postgresdb_restore_enabled = true + postgresdb_restore_config = { + bucket_uri = "s3://my-backup-dumps-databases" #S3 bucket URI (without a trailing slash /) containing the backup dump file. + backup_file_name = "atmosly_db1.sql" #Give .sql or .zip file for restore + } } diff --git a/examples/complete/output.tf b/examples/complete/output.tf index f7ef643..e6fd035 100644 --- a/examples/complete/output.tf +++ b/examples/complete/output.tf @@ -12,6 +12,11 @@ output "instance_name" { value = module.rds-pg.db_instance_name } +output "db_name" { + description = "Database name" + value = module.rds-pg.db_name +} + output "rds-mysql_replica_db_instance_name" { description = "The name of the database instance" value = module.rds-pg.replica_db_instance_name diff --git a/examples/complete/provider.tf b/examples/complete/provider.tf index 369af88..4b221da 100644 --- a/examples/complete/provider.tf +++ b/examples/complete/provider.tf @@ -4,3 +4,24 @@ provider "aws" { tags = local.additional_tags } } +data "aws_eks_cluster" "cluster" { + name = local.cluster_name + +} +data "aws_eks_cluster_auth" "cluster" { + name = local.cluster_name +} + +provider "kubernetes" { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.cluster.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.cluster.token + } +} diff --git a/examples/complete/version.tf b/examples/complete/version.tf index 515c2d2..d8e2113 100644 --- a/examples/complete/version.tf +++ b/examples/complete/version.tf @@ -3,7 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 3.43.0" + version = ">= 5.0.0" } } } diff --git a/helm/values/backup/values.yaml b/helm/values/backup/values.yaml new file mode 100644 index 0000000..ae304b5 --- /dev/null +++ b/helm/values/backup/values.yaml @@ -0,0 +1,34 @@ +## Enable Full backup +backup: + bucket_uri: ${bucket_uri} + cron_for_full_backup: "${cron_for_full_backup}" + postgres_database_name: "${postgres_database_name}" + database_endpoint: "${db_endpoint}" + database_password: "${db_password}" + database_user: "${db_username}" + + +annotations: + ${annotations} + +auth: + username: "${db_username}" + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +backupjob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/helm/values/restore/values.yaml b/helm/values/restore/values.yaml new file mode 100644 index 0000000..13dbf8d --- /dev/null +++ b/helm/values/restore/values.yaml @@ -0,0 +1,31 @@ +restore: + bucket_uri: ${bucket_uri} + db_endpoint: "${db_endpoint}" + db_password: "${db_password}" + db_username: "${db_username}" + backup_file_name: "${backup_file_name}" + +auth: + username: "${db_username}" + +annotations: + ${annotations} + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "Addons-Services" + operator: In + values: + - "true" + +restorejob: + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 200Mi + cpu: 100m diff --git a/main.tf b/main.tf index bc343f3..a85d905 100644 --- a/main.tf +++ b/main.tf @@ -2,6 +2,11 @@ data "aws_region" "current" {} data "aws_availability_zones" "available" {} locals { + + db_password = var.custom_user_password != "" ? var.custom_user_password : ( + length(random_password.master) > 0 ? element(random_password.master, 0).result : var.custom_user_password + ) + tags = { Automation = "true" Environment = var.environment @@ -131,7 +136,7 @@ resource "aws_security_group_rule" "cidr_ingress" { module "security_group_rds" { source = "terraform-aws-modules/security-group/aws" - version = "~> 4" + version = "~> 5.0" name = format("%s-%s-%s", var.environment, var.name, "rds-sg") create = var.create_security_group vpc_id = var.vpc_id @@ -155,7 +160,7 @@ module "security_group_rds" { resource "aws_secretsmanager_secret" "secret_master_db" { name = format("%s/%s/%s", var.environment, var.name, "rds-postgresql-pass") tags = merge( - { "Name" = format("%s/%s/%s", var.environment, var.name, "rds-mysql-pass") }, + { "Name" = format("%s/%s/%s", var.environment, var.name, "rds-postgres-pass") }, local.tags, ) } @@ -167,16 +172,13 @@ resource "random_password" "master" { } resource "aws_secretsmanager_secret_version" "rds_credentials" { - count = length(random_password.master) > 0 ? 1 : 0 - secret_id = aws_secretsmanager_secret.secret_master_db.id - secret_string = < 0 ? element(random_password.master, 0).result : var.custom_password, - "engine": "${var.engine}", - "host": "${module.db.db_instance_endpoint}" -} -EOF + secret_id = aws_secretsmanager_secret.secret_master_db.id + secret_string = jsonencode({ + username = module.db.db_instance_username + password = local.db_password + engine = var.engine + host = module.db.db_instance_endpoint + }) } # Cloudwatch alarms @@ -317,3 +319,30 @@ resource "aws_lambda_permission" "sns_lambda_slack_invoke" { principal = "sns.amazonaws.com" source_arn = aws_sns_topic.slack_topic[0].arn } + +module "backup_restore" { + depends_on = [module.db] + source = "./modules/db-backup-restore" + name = var.name + cluster_name = var.cluster_name + namespace = var.namespace + create_namespace = var.create_namespace + postgresdb_backup_enabled = var.postgresdb_backup_enabled + postgresdb_backup_config = { + db_username = var.master_username + db_password = var.custom_user_password != "" ? var.custom_user_password : nonsensitive(random_password.master[0].result) + postgres_database_name = var.postgresdb_backup_config.postgres_database_name + cron_for_full_backup = var.postgresdb_backup_config.cron_for_full_backup + bucket_uri = var.postgresdb_backup_config.bucket_uri + db_endpoint = replace(var.replica_enable ? module.db_replica[0].db_instance_endpoint : module.db.db_instance_endpoint, ":5432", "") + } + + postgresdb_restore_enabled = var.postgresdb_restore_enabled + postgresdb_restore_config = { + db_endpoint = replace(var.replica_enable ? module.db_replica[0].db_instance_endpoint : module.db.db_instance_endpoint, ":5432", "") + db_username = var.master_username + db_password = var.custom_user_password != "" ? var.custom_user_password : nonsensitive(random_password.master[0].result) + bucket_uri = var.postgresdb_restore_config.bucket_uri + backup_file_name = var.postgresdb_restore_config.backup_file_name, + } +} diff --git a/modules/db-backup-restore/README.md b/modules/db-backup-restore/README.md new file mode 100644 index 0000000..8283070 --- /dev/null +++ b/modules/db-backup-restore/README.md @@ -0,0 +1,57 @@ +# db-backup-restore + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [helm](#provider\_helm) | n/a | +| [kubernetes](#provider\_kubernetes) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.postgres_backup_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.postgres_restore_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [helm_release.postgresdb_backup](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.postgresdb_restore](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace.postgresdb](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.kubernetes_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [azure\_container\_name](#input\_azure\_container\_name) | Azure container name | `string` | `""` | no | +| [azure\_storage\_account\_key](#input\_azure\_storage\_account\_key) | Azure storage account key | `string` | `""` | no | +| [azure\_storage\_account\_name](#input\_azure\_storage\_account\_name) | Azure storage account name | `string` | `""` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | +| [iam\_role\_arn\_backup](#input\_iam\_role\_arn\_backup) | IAM role ARN for backup (AWS) | `string` | `""` | no | +| [iam\_role\_arn\_restore](#input\_iam\_role\_arn\_restore) | IAM role ARN for restore (AWS) | `string` | `""` | no | +| [name](#input\_name) | Name identifier for module to be added as suffix to resources | `string` | `"test"` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"postgresdb"` | no | +| [postgresdb\_backup\_config](#input\_postgresdb\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` | 
{
  "bucket_uri": "",
  "cron_for_full_backup": "",
  "postgres_database_name": "",
  "s3_bucket_region": ""
}
| no | +| [postgresdb\_backup\_enabled](#input\_postgresdb\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [postgresdb\_permission](#input\_postgresdb\_permission) | access | `bool` | `false` | no | +| [postgresdb\_restore\_config](#input\_postgresdb\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` | 
{
  "DB_NAME": "",
  "backup_file_name": "",
  "bucket_uri": "",
  "file_name": ""
}
| no | +| [postgresdb\_restore\_enabled](#input\_postgresdb\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | +| [service\_account\_backup](#input\_service\_account\_backup) | Service account for backup (GCP) | `string` | `""` | no | +| [service\_account\_restore](#input\_service\_account\_restore) | Service account for restore (GCP) | `string` | `""` | no | + +## Outputs + +No outputs. + diff --git a/modules/db-backup-restore/backup/Chart.yaml b/modules/db-backup-restore/backup/Chart.yaml new file mode 100644 index 0000000..38491ed --- /dev/null +++ b/modules/db-backup-restore/backup/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A helm chart for Backup of postgres and stored in S3 +name: postgres-backup +version: 1.0.0 diff --git a/modules/db-backup-restore/backup/templates/backup-secret.yaml b/modules/db-backup-restore/backup/templates/backup-secret.yaml new file mode 100644 index 0000000..18dbb52 --- /dev/null +++ b/modules/db-backup-restore/backup/templates/backup-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-bucket-uri + namespace: {{ .Release.Namespace }} + labels: +data: + MYSQL_BUCKET_URI: {{ .Values.backup.bucket_uri | b64enc | quote }} diff --git a/modules/db-backup-restore/backup/templates/cronjob.yaml b/modules/db-backup-restore/backup/templates/cronjob.yaml new file mode 100644 index 0000000..357b72e --- /dev/null +++ b/modules/db-backup-restore/backup/templates/cronjob.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: backup-postgresdb +spec: + schedule: {{ .Values.backup.cron_for_full_backup | quote }} + concurrencyPolicy: Forbid + suspend: false + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + + jobTemplate: + spec: + template: + spec: + affinity: {{ .Values.affinity | toYaml | nindent 10 }} + restartPolicy: OnFailure + serviceAccountName: sa-postgres-backup + containers: + - name: backup-postgresdb + image: squareops01/rds-postgresql-backup:v2 + imagePullPolicy: Always + command: ["/backup/backup_script.sh"] + env: + - name: DB_HOST + value: {{ .Values.backup.database_endpoint }} + - name: DB_USER + value: {{ .Values.backup.database_user }} + - name: DB_PASSWORD + value: {{ .Values.backup.database_password }} + - name: S3_BUCKET + value: {{ .Values.backup.bucket_uri }} + - name: DB_PORT + value: "5432" + - name: DB_NAME + value: {{ .Values.backup.postgres_database_name }} + resources: {{ .Values.backupjob.resources | toYaml | nindent 12 }} diff --git a/modules/db-backup-restore/backup/templates/service_account.yaml b/modules/db-backup-restore/backup/templates/service_account.yaml new file mode 100644 index 0000000..e1c79e3 --- /dev/null +++ b/modules/db-backup-restore/backup/templates/service_account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sa-postgres-backup + namespace: {{ .Release.Namespace }} + annotations: + {{ toYaml .Values.annotations | indent 4 }} diff --git a/modules/db-backup-restore/main.tf b/modules/db-backup-restore/main.tf new file mode 100644 index 0000000..472c2bc --- /dev/null +++ b/modules/db-backup-restore/main.tf @@ -0,0 +1,49 @@ +resource "kubernetes_namespace" "postgresdb" { + count = var.create_namespace ? 1 : 0 + metadata { + annotations = {} + name = var.namespace + } +} + +resource "helm_release" "postgresdb_backup" { + count = var.postgresdb_backup_enabled ? 1 : 0 + depends_on = [kubernetes_namespace.postgresdb] + name = "postgresdb-backup" + chart = "../../modules/db-backup-restore/backup" + timeout = 600 + namespace = var.namespace + values = [ + templatefile("${path.module}/../../helm/values/backup/values.yaml", { + bucket_uri = var.postgresdb_backup_config.bucket_uri, + postgres_database_name = var.postgresdb_backup_config.postgres_database_name, + db_endpoint = var.postgresdb_backup_config.db_endpoint, + db_password = var.postgresdb_backup_config.db_password, + db_username = var.postgresdb_backup_config.db_username, + # s3_bucket_region = var.postgresdb_backup_config.s3_bucket_region , + cron_for_full_backup = var.postgresdb_backup_config.cron_for_full_backup, + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.postgres_backup_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}" + }) + ] +} + + +## DB dump restore +resource "helm_release" "postgresdb_restore" { + count = var.postgresdb_restore_enabled ? 1 : 0 + depends_on = [kubernetes_namespace.postgresdb] + name = "postgresdb-restore" + chart = "../../modules/db-backup-restore/restore" + timeout = 600 + namespace = var.namespace + values = [ + templatefile("${path.module}/../../helm/values/restore/values.yaml", { + bucket_uri = var.postgresdb_restore_config.bucket_uri, + db_endpoint = var.postgresdb_restore_config.db_endpoint, + db_password = var.postgresdb_restore_config.db_password, + db_username = var.postgresdb_restore_config.db_username, + backup_file_name = var.postgresdb_restore_config.backup_file_name, + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.postgres_restore_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}" + }) + ] +} diff --git a/modules/db-backup-restore/restore/Chart.yaml b/modules/db-backup-restore/restore/Chart.yaml new file mode 100644 index 0000000..2badf7b --- /dev/null +++ b/modules/db-backup-restore/restore/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A helm chart for restore of postgres and stored in S3 +name: postgres-restore +version: 1.0.0 diff --git a/modules/db-backup-restore/restore/templates/job.yaml b/modules/db-backup-restore/restore/templates/job.yaml new file mode 100644 index 0000000..9b3cdb6 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/job.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: restore +spec: + template: + spec: + affinity: {{ .Values.affinity | toYaml | nindent 6 }} + serviceAccountName: sa-postgres-restore + containers: + - name: restore-postgresdb + image: squareops01/rds-postgresql-restore:v2 + imagePullPolicy: Always + command: ["/restore/restore_script.sh"] + env: + - name: DB_HOST + value: {{ .Values.restore.db_endpoint }} + - name: DB_USER + value: {{ .Values.restore.db_username }} + - name: DB_PASSWORD + value: {{ .Values.restore.db_password }} + - name: POSTGRESQL_BUCKET_RESTORE_URI + value: {{ .Values.restore.bucket_uri }} + - name: DB_PORT + value: "5432" + - name: RESTORE_FILE_NAME + value: {{ .Values.restore.backup_file_name }} + resources: {{ .Values.restorejob.resources | toYaml | nindent 12 }} + restartPolicy: Never + backoffLimit: 4 diff --git a/modules/db-backup-restore/restore/templates/restore-secret.yaml b/modules/db-backup-restore/restore/templates/restore-secret.yaml new file mode 100644 index 0000000..35b0c98 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/restore-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-bucket-uri-restore + namespace: {{ .Release.Namespace }} + labels: +data: + MYSQL_BUCKET_URI: {{ .Values.restore.bucket_uri | b64enc | quote }} diff --git a/modules/db-backup-restore/restore/templates/service_account.yaml b/modules/db-backup-restore/restore/templates/service_account.yaml new file mode 100644 index 0000000..9af74f0 --- /dev/null +++ b/modules/db-backup-restore/restore/templates/service_account.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sa-postgres-restore + annotations: + {{ toYaml .Values.annotations | indent 4 }} diff --git a/modules/db-backup-restore/roles.tf b/modules/db-backup-restore/roles.tf new file mode 100644 index 0000000..8a8b17a --- /dev/null +++ b/modules/db-backup-restore/roles.tf @@ -0,0 +1,100 @@ +locals { + oidc_provider = replace( + data.aws_eks_cluster.kubernetes_cluster.identity[0].oidc[0].issuer, + "/^https:///", + "" + ) +} + +data "aws_caller_identity" "current" {} + +data "aws_eks_cluster" "kubernetes_cluster" { + name = var.cluster_name +} + +resource "aws_iam_role" "postgres_backup_role" { + count = var.postgresdb_backup_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-backup-rds") + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}" + }, + Action = "sts:AssumeRoleWithWebIdentity", + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com", + "${local.oidc_provider}:sub" = "system:serviceaccount:${var.namespace}:sa-postgres-backup" + } + } + } + ] + }) + inline_policy { + name = "AllowS3PutObject" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + Effect = "Allow" + Resource = "*" + } + ] + }) + } +} + + +resource "aws_iam_role" "postgres_restore_role" { + count = var.postgresdb_restore_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-restore") + assume_role_policy = jsonencode({ + Version = "2012-10-17", + Statement = [ + { + Effect = "Allow", + Principal = { + Federated = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${local.oidc_provider}" + }, + Action = "sts:AssumeRoleWithWebIdentity", + Condition = { + StringEquals = { + "${local.oidc_provider}:aud" = "sts.amazonaws.com", + "${local.oidc_provider}:sub" = "system:serviceaccount:${var.namespace}:sa-postgres-restore" + } + } + } + ] + }) + inline_policy { + name = "AllowS3PutObject" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + Effect = "Allow" + Resource = "*" + } + ] + }) + } +} diff --git a/modules/db-backup-restore/variables.tf b/modules/db-backup-restore/variables.tf new file mode 100644 index 0000000..1a23f03 --- /dev/null +++ b/modules/db-backup-restore/variables.tf @@ -0,0 +1,115 @@ +variable "iam_role_arn_backup" { + description = "IAM role ARN for backup (AWS)" + type = string + default = "" +} + +variable "service_account_backup" { + description = "Service account for backup (GCP)" + type = string + default = "" +} + +variable "azure_storage_account_name" { + description = "Azure storage account name" + type = string + default = "" +} + +variable "azure_storage_account_key" { + description = "Azure storage account key" + type = string + default = "" +} + +variable "azure_container_name" { + description = "Azure container name" + type = string + default = "" +} + +variable "namespace" { + type = string + default = "postgresdb" + description = "Name of the Kubernetes namespace where the MYSQL deployment will be deployed." +} + +variable "create_namespace" { + type = string + description = "Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace." + default = false +} + +variable "iam_role_arn_restore" { + description = "IAM role ARN for restore (AWS)" + type = string + default = "" +} + +variable "service_account_restore" { + description = "Service account for restore (GCP)" + type = string + default = "" +} + +# two variable of clustername and name +variable "name" { + description = "Name identifier for module to be added as suffix to resources" + type = string + default = "test" +} + +variable "cluster_name" { + type = string + default = "" + description = "Specifies the name of the EKS cluster to deploy the MySQL application on." +} + +variable "postgresdb_permission" { + default = false + description = "access" + type = bool +} + +variable "bucket_provider_type" { + type = string + default = "s3" + description = "Choose what type of provider you want (s3, gcs)" +} + + +variable "postgresdb_backup_enabled" { + type = bool + default = false + description = "Specifies whether to enable backups for MySQL database." +} + +variable "postgresdb_restore_enabled" { + type = bool + default = false + description = "Specifies whether to enable restoring dump to the MySQL database." +} + +variable "postgresdb_backup_config" { + type = map(string) + default = { + bucket_uri = "" + s3_bucket_region = "" + cron_for_full_backup = "" + postgres_database_name = "" + # db_endpoint="" + } + description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." +} + +variable "postgresdb_restore_config" { + type = any + default = { + bucket_uri = "" + file_name = "" + # s3_bucket_region = "" + DB_NAME = "" + backup_file_name = "" + } + description = "Configuration options for restoring dump to the MySQL database." +} diff --git a/outputs.tf b/outputs.tf index eefb182..dd0df87 100644 --- a/outputs.tf +++ b/outputs.tf @@ -23,6 +23,11 @@ output "db_instance_username" { value = nonsensitive(module.db.db_instance_username) } +output "db_name" { + description = "The database name used in the RDS module" + value = var.db_name +} + output "db_instance_password" { description = "Password for accessing the database." value = var.custom_user_password != "" ? var.custom_user_password : nonsensitive(random_password.master[0].result) diff --git a/variables.tf b/variables.tf index 6ecf7c4..db9bf05 100644 --- a/variables.tf +++ b/variables.tf @@ -311,3 +311,62 @@ variable "custom_user_password" { default = "" type = string } + +variable "create_namespace" { + type = string + description = "Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace." + default = false +} + +variable "namespace" { + type = string + default = "postgresdb" + description = "Name of the Kubernetes namespace where the MYSQL deployment will be deployed." +} + +variable "postgresdb_backup_enabled" { + type = bool + default = false + description = "Specifies whether to enable backups for MySQL database." +} + +variable "postgresdb_restore_enabled" { + type = bool + default = false + description = "Specifies whether to enable restoring dump to the MySQL database." +} + + +variable "bucket_provider_type" { + type = string + default = "s3" + description = "Choose what type of provider you want (s3, gcs)" +} + +variable "postgresdb_backup_config" { + type = map(string) + default = { + bucket_uri = "" + # s3_bucket_region = "" + cron_for_full_backup = "" + postgres_database_name = "" + # db_endpoint="" + } + description = "configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups." +} + +variable "postgresdb_restore_config" { + type = any + default = { + bucket_uri = "" + file_name = "" + # s3_bucket_region = "" + } + description = "Configuration options for restoring dump to the MySQL database." +} + +variable "cluster_name" { + type = string + default = "" + description = "Specifies the name of the EKS cluster to deploy the MySQL application on." +}