From aa63d18628f2c8c372c664e20cc159637dc368b0 Mon Sep 17 00:00:00 2001 From: amanravi-squareops Date: Fri, 7 Feb 2025 17:02:49 +0530 Subject: [PATCH 1/2] fixed restore database --- examples/complete/main.tf | 29 ++++++++++--------- examples/complete/provider.tf | 4 +-- main.tf | 18 +++++------- .../backup/templates/cronjob.yaml | 2 +- modules/db-backup-restore/main.tf | 4 +-- .../restore/templates/job.yaml | 2 +- modules/db-backup-restore/roles.tf | 6 ++-- 7 files changed, 33 insertions(+), 32 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 2cec762..cc4c5cc 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -6,7 +6,7 @@ locals { environment = "prod" create_namespace = true namespace = "pg" - engine_version = "15.4" + engine_version = "15.7" instance_class = "db.t4g.micro" storage_type = "gp3" cluster_name = "" @@ -92,7 +92,7 @@ module "vpc" { name = local.name vpc_cidr = local.vpc_cidr environment = local.environment - availability_zones = ["us-east-1a", "us-east-1b"] + availability_zones = ["us-east-2a", "us-east-2b"] public_subnet_enabled = true auto_assign_public_ip = true intra_subnet_enabled = false @@ -104,14 +104,14 @@ module "vpc" { module "rds-pg" { source = "squareops/rds-postgresql/aws" version = "2.0.0" - name = local.name - db_name = "test" - multi_az = false - family = local.family + name = local.name + db_name = "test" + multi_az = false + family = local.family vpc_id = module.vpc.vpc_id - allowed_security_groups = local.allowed_security_groups + allowed_security_groups = local.allowed_security_groups subnet_ids = module.vpc.database_subnets - environment = local.environment + environment = local.environment kms_key_arn = module.kms.key_arn storage_type = local.storage_type engine_version = local.engine_version @@ -141,13 +141,14 @@ module "rds-pg" { create_namespace = local.create_namespace postgresdb_backup_enabled = false postgresdb_backup_config = { - postgres_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases - cron_for_full_backup = "*/2 * * * *" # set cronjob for backup - bucket_uri = "s3://postgres-backups-atmosly" # s3 bucket uri + postgres_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases + cron_for_full_backup = "*/2 * * * *" # set cronjob for backup + bucket_uri = "s3://my-backup-dumps-databases" # s3 bucket uri } - postgresdb_restore_enabled = false + postgresdb_restore_enabled = true postgresdb_restore_config = { - bucket_uri = "s3://postgres-backups-atmosly" #S3 bucket URI (without a trailing slash /) containing the backup dump file. - backup_file_name = "db5_20241114111607.sql" #Give .sql or .zip file for restore + bucket_uri = "s3://my-backup-dumps-databases" #S3 bucket URI (without a trailing slash /) containing the backup dump file. + backup_file_name = "atmosly_db1.sql" #Give .sql or .zip file for restore } } + diff --git a/examples/complete/provider.tf b/examples/complete/provider.tf index af8c494..4b221da 100644 --- a/examples/complete/provider.tf +++ b/examples/complete/provider.tf @@ -5,11 +5,11 @@ provider "aws" { } } data "aws_eks_cluster" "cluster" { - name = "" + name = local.cluster_name } data "aws_eks_cluster_auth" "cluster" { - name = "" + name = local.cluster_name } provider "kubernetes" { diff --git a/main.tf b/main.tf index 9aa3795..a85d905 100644 --- a/main.tf +++ b/main.tf @@ -323,6 +323,7 @@ resource "aws_lambda_permission" "sns_lambda_slack_invoke" { module "backup_restore" { depends_on = [module.db] source = "./modules/db-backup-restore" + name = var.name cluster_name = var.cluster_name namespace = var.namespace create_namespace = var.create_namespace @@ -331,20 +332,17 @@ module "backup_restore" { db_username = var.master_username db_password = var.custom_user_password != "" ? var.custom_user_password : nonsensitive(random_password.master[0].result) postgres_database_name = var.postgresdb_backup_config.postgres_database_name - # s3_bucket_region = var.postgresdb_backup_config.s3_bucket_region - cron_for_full_backup = var.postgresdb_backup_config.cron_for_full_backup - bucket_uri = var.postgresdb_backup_config.bucket_uri - db_endpoint = replace(var.replica_enable ? module.db_replica[0].db_instance_endpoint : module.db.db_instance_endpoint, ":5432", "") + cron_for_full_backup = var.postgresdb_backup_config.cron_for_full_backup + bucket_uri = var.postgresdb_backup_config.bucket_uri + db_endpoint = replace(var.replica_enable ? module.db_replica[0].db_instance_endpoint : module.db.db_instance_endpoint, ":5432", "") } postgresdb_restore_enabled = var.postgresdb_restore_enabled postgresdb_restore_config = { - db_endpoint = replace(var.replica_enable ? module.db_replica[0].db_instance_endpoint : module.db.db_instance_endpoint, ":5432", "") - db_username = var.master_username - db_password = var.custom_user_password != "" ? var.custom_user_password : nonsensitive(random_password.master[0].result) - bucket_uri = var.postgresdb_restore_config.bucket_uri - # s3_bucket_region = var.postgresdb_restore_config.s3_bucket_region - # DB_NAME = var.postgresdb_restore_config.DB_NAME, + db_endpoint = replace(var.replica_enable ? module.db_replica[0].db_instance_endpoint : module.db.db_instance_endpoint, ":5432", "") + db_username = var.master_username + db_password = var.custom_user_password != "" ? var.custom_user_password : nonsensitive(random_password.master[0].result) + bucket_uri = var.postgresdb_restore_config.bucket_uri backup_file_name = var.postgresdb_restore_config.backup_file_name, } } diff --git a/modules/db-backup-restore/backup/templates/cronjob.yaml b/modules/db-backup-restore/backup/templates/cronjob.yaml index 01a70fc..357b72e 100644 --- a/modules/db-backup-restore/backup/templates/cronjob.yaml +++ b/modules/db-backup-restore/backup/templates/cronjob.yaml @@ -18,7 +18,7 @@ spec: serviceAccountName: sa-postgres-backup containers: - name: backup-postgresdb - image: squareops01/rds-postgresql-backup:v1 + image: squareops01/rds-postgresql-backup:v2 imagePullPolicy: Always command: ["/backup/backup_script.sh"] env: diff --git a/modules/db-backup-restore/main.tf b/modules/db-backup-restore/main.tf index ebf3fcd..472c2bc 100644 --- a/modules/db-backup-restore/main.tf +++ b/modules/db-backup-restore/main.tf @@ -22,7 +22,7 @@ resource "helm_release" "postgresdb_backup" { db_username = var.postgresdb_backup_config.db_username, # s3_bucket_region = var.postgresdb_backup_config.s3_bucket_region , cron_for_full_backup = var.postgresdb_backup_config.cron_for_full_backup, - annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.postgres_backup_role.arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}" + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.postgres_backup_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_backup}" }) ] } @@ -43,7 +43,7 @@ resource "helm_release" "postgresdb_restore" { db_password = var.postgresdb_restore_config.db_password, db_username = var.postgresdb_restore_config.db_username, backup_file_name = var.postgresdb_restore_config.backup_file_name, - annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.postgres_restore_role.arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}" + annotations = var.bucket_provider_type == "s3" ? "eks.amazonaws.com/role-arn: ${aws_iam_role.postgres_restore_role[count.index].arn}" : "iam.gke.io/gcp-service-account: ${var.service_account_restore}" }) ] } diff --git a/modules/db-backup-restore/restore/templates/job.yaml b/modules/db-backup-restore/restore/templates/job.yaml index 473b3ee..9b3cdb6 100644 --- a/modules/db-backup-restore/restore/templates/job.yaml +++ b/modules/db-backup-restore/restore/templates/job.yaml @@ -9,7 +9,7 @@ spec: serviceAccountName: sa-postgres-restore containers: - name: restore-postgresdb - image: squareops01/rds-postgresql-restore:v1 + image: squareops01/rds-postgresql-restore:v2 imagePullPolicy: Always command: ["/restore/restore_script.sh"] env: diff --git a/modules/db-backup-restore/roles.tf b/modules/db-backup-restore/roles.tf index 0abc253..8302030 100644 --- a/modules/db-backup-restore/roles.tf +++ b/modules/db-backup-restore/roles.tf @@ -13,7 +13,8 @@ data "aws_eks_cluster" "kubernetes_cluster" { } resource "aws_iam_role" "postgres_backup_role" { - name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-backup-rds") + count = var.postgresdb_backup_enabled ? 1 : 0 + name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-backup-rds") assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [ @@ -56,6 +57,7 @@ resource "aws_iam_role" "postgres_backup_role" { resource "aws_iam_role" "postgres_restore_role" { + count = var.postgresdb_restore_enabled ? 1 : 0 name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-restore") assume_role_policy = jsonencode({ Version = "2012-10-17", @@ -95,4 +97,4 @@ resource "aws_iam_role" "postgres_restore_role" { ] }) } -} +} From a8b75b383f133188ceac83832e79c9077cd04766 Mon Sep 17 00:00:00 2001 From: amanravi-squareops Date: Fri, 7 Feb 2025 17:13:01 +0530 Subject: [PATCH 2/2] run lint command --- README.md | 1 + examples/complete/README.md | 3 +- examples/complete/main.tf | 15 ++++---- modules/db-backup-restore/README.md | 57 +++++++++++++++++++++++++++++ modules/db-backup-restore/roles.tf | 2 +- 5 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 modules/db-backup-restore/README.md diff --git a/README.md b/README.md index ce0993a..9038012 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,7 @@ The required IAM permissions to create resources from this module can be found [ | [db\_instance\_name](#output\_db\_instance\_name) | Name of the database instance | | [db\_instance\_password](#output\_db\_instance\_password) | Password for accessing the database. | | [db\_instance\_username](#output\_db\_instance\_username) | Master username for accessing the database. | +| [db\_name](#output\_db\_name) | The database name used in the RDS module | | [db\_parameter\_group\_id](#output\_db\_parameter\_group\_id) | ID of the parameter group associated with the RDS instance. | | [db\_subnet\_group\_id](#output\_db\_subnet\_group\_id) | ID of the subnet group associated with the RDS instance. | | [master\_credential\_secret\_arn](#output\_master\_credential\_secret\_arn) | The ARN of the master user secret (Only available when manage\_master\_user\_password is set to true) | diff --git a/examples/complete/README.md b/examples/complete/README.md index b738988..9b26e16 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -27,7 +27,7 @@ This example will be very useful for users who are new to a module and want to q | Name | Source | Version | |------|--------|---------| | [kms](#module\_kms) | terraform-aws-modules/kms/aws | ~> 1.0 | -| [rds-pg](#module\_rds-pg) | squareops/rds-postgresql/aws | n/a | +| [rds-pg](#module\_rds-pg) | squareops/rds-postgresql/aws | 2.0.0 | | [vpc](#module\_vpc) | squareops/vpc/aws | n/a | ## Resources @@ -47,6 +47,7 @@ No inputs. | Name | Description | |------|-------------| +| [db\_name](#output\_db\_name) | Database name | | [instance\_endpoint](#output\_instance\_endpoint) | Connection endpoint of the RDS instance. | | [instance\_name](#output\_instance\_name) | Name of the database instance. | | [instance\_password](#output\_instance\_password) | Password for accessing the database (Note: Terraform does not track this password after initial creation). | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index cc4c5cc..6a5c226 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -104,14 +104,14 @@ module "vpc" { module "rds-pg" { source = "squareops/rds-postgresql/aws" version = "2.0.0" - name = local.name - db_name = "test" - multi_az = false - family = local.family + name = local.name + db_name = "test" + multi_az = false + family = local.family vpc_id = module.vpc.vpc_id - allowed_security_groups = local.allowed_security_groups + allowed_security_groups = local.allowed_security_groups subnet_ids = module.vpc.database_subnets - environment = local.environment + environment = local.environment kms_key_arn = module.kms.key_arn storage_type = local.storage_type engine_version = local.engine_version @@ -141,7 +141,7 @@ module "rds-pg" { create_namespace = local.create_namespace postgresdb_backup_enabled = false postgresdb_backup_config = { - postgres_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases + postgres_database_name = "" # Specify the database name or Leave empty if you wish to backup all databases cron_for_full_backup = "*/2 * * * *" # set cronjob for backup bucket_uri = "s3://my-backup-dumps-databases" # s3 bucket uri } @@ -151,4 +151,3 @@ module "rds-pg" { backup_file_name = "atmosly_db1.sql" #Give .sql or .zip file for restore } } - diff --git a/modules/db-backup-restore/README.md b/modules/db-backup-restore/README.md new file mode 100644 index 0000000..8283070 --- /dev/null +++ b/modules/db-backup-restore/README.md @@ -0,0 +1,57 @@ +# db-backup-restore + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | n/a | +| [helm](#provider\_helm) | n/a | +| [kubernetes](#provider\_kubernetes) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.postgres_backup_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.postgres_restore_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [helm_release.postgresdb_backup](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.postgresdb_restore](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace.postgresdb](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_eks_cluster.kubernetes_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [azure\_container\_name](#input\_azure\_container\_name) | Azure container name | `string` | `""` | no | +| [azure\_storage\_account\_key](#input\_azure\_storage\_account\_key) | Azure storage account key | `string` | `""` | no | +| [azure\_storage\_account\_name](#input\_azure\_storage\_account\_name) | Azure storage account name | `string` | `""` | no | +| [bucket\_provider\_type](#input\_bucket\_provider\_type) | Choose what type of provider you want (s3, gcs) | `string` | `"s3"` | no | +| [cluster\_name](#input\_cluster\_name) | Specifies the name of the EKS cluster to deploy the MySQL application on. | `string` | `""` | no | +| [create\_namespace](#input\_create\_namespace) | Specify whether or not to create the namespace if it does not already exist. Set it to true to create the namespace. | `string` | `false` | no | +| [iam\_role\_arn\_backup](#input\_iam\_role\_arn\_backup) | IAM role ARN for backup (AWS) | `string` | `""` | no | +| [iam\_role\_arn\_restore](#input\_iam\_role\_arn\_restore) | IAM role ARN for restore (AWS) | `string` | `""` | no | +| [name](#input\_name) | Name identifier for module to be added as suffix to resources | `string` | `"test"` | no | +| [namespace](#input\_namespace) | Name of the Kubernetes namespace where the MYSQL deployment will be deployed. | `string` | `"postgresdb"` | no | +| [postgresdb\_backup\_config](#input\_postgresdb\_backup\_config) | configuration options for MySQL database backups. It includes properties such as the S3 bucket URI, the S3 bucket region, and the cron expression for full backups. | `map(string)` | 
{
  "bucket_uri": "",
  "cron_for_full_backup": "",
  "postgres_database_name": "",
  "s3_bucket_region": ""
}
| no | +| [postgresdb\_backup\_enabled](#input\_postgresdb\_backup\_enabled) | Specifies whether to enable backups for MySQL database. | `bool` | `false` | no | +| [postgresdb\_permission](#input\_postgresdb\_permission) | access | `bool` | `false` | no | +| [postgresdb\_restore\_config](#input\_postgresdb\_restore\_config) | Configuration options for restoring dump to the MySQL database. | `any` | 
{
  "DB_NAME": "",
  "backup_file_name": "",
  "bucket_uri": "",
  "file_name": ""
}
| no | +| [postgresdb\_restore\_enabled](#input\_postgresdb\_restore\_enabled) | Specifies whether to enable restoring dump to the MySQL database. | `bool` | `false` | no | +| [service\_account\_backup](#input\_service\_account\_backup) | Service account for backup (GCP) | `string` | `""` | no | +| [service\_account\_restore](#input\_service\_account\_restore) | Service account for restore (GCP) | `string` | `""` | no | + +## Outputs + +No outputs. + diff --git a/modules/db-backup-restore/roles.tf b/modules/db-backup-restore/roles.tf index 8302030..8a8b17a 100644 --- a/modules/db-backup-restore/roles.tf +++ b/modules/db-backup-restore/roles.tf @@ -58,7 +58,7 @@ resource "aws_iam_role" "postgres_backup_role" { resource "aws_iam_role" "postgres_restore_role" { count = var.postgresdb_restore_enabled ? 1 : 0 - name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-restore") + name = format("%s-%s-%s", var.cluster_name, var.name, "postgres-restore") assume_role_policy = jsonencode({ Version = "2012-10-17", Statement = [