From 5ba916070a8da913201bb0e4a4eb694cdacc012c Mon Sep 17 00:00:00 2001 From: Caio Quirino Date: Wed, 17 Jul 2019 15:48:56 +0200 Subject: [PATCH] Added tf_data_dir parameter and changed plugin to respect TF_DATA_DIR This commit will enable the plugin to do parallel builds. The first change is to use Terraform's TF_DATA_DIR environment variable to change the state directory so they will not conflict each other in parallel executions The second change is to change the plan's output to a different file so they will not be overriden by another parallel execution --- main.go | 30 +++++++------ plugin.go | 114 +++++++++++++++++++++++++++++++++---------------- plugin_test.go | 55 ++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 49 deletions(-) diff --git a/main.go b/main.go index 20e9d3c..23e6f7d 100644 --- a/main.go +++ b/main.go @@ -108,6 +108,11 @@ func main() { Usage: "a list of var files to use. Each value is passed as -var-file=", EnvVar: "PLUGIN_VAR_FILES", }, + cli.StringSliceFlag{ + Name: "tf_data_dir", + Usage: "changes the location where Terraform keeps its per-working-directory data, such as the current remote backend configuration.", + EnvVar: "PLUGIN_TF_DATA_DIR", + }, } if err := app.Run(os.Args); err != nil { @@ -144,18 +149,19 @@ func run(c *cli.Context) error { plugin := Plugin{ Config: Config{ - Actions: c.StringSlice("actions"), - Vars: vars, - Secrets: secrets, - InitOptions: initOptions, - FmtOptions: fmtOptions, - Cacert: c.String("ca_cert"), - Sensitive: c.Bool("sensitive"), - RoleARN: c.String("role_arn_to_assume"), - RootDir: c.String("root_dir"), - Parallelism: c.Int("parallelism"), - Targets: c.StringSlice("targets"), - VarFiles: c.StringSlice("var_files"), + Actions: c.StringSlice("actions"), + Vars: vars, + Secrets: secrets, + InitOptions: initOptions, + FmtOptions: fmtOptions, + Cacert: c.String("ca_cert"), + Sensitive: c.Bool("sensitive"), + RoleARN: c.String("role_arn_to_assume"), + RootDir: c.String("root_dir"), + Parallelism: c.Int("parallelism"), + Targets: c.StringSlice("targets"), + VarFiles: c.StringSlice("var_files"), + TerraformDataDir: c.String("tf_data_dir"), }, Netrc: Netrc{ Login: c.String("netrc.username"), diff --git a/plugin.go b/plugin.go index 6fefc24..bea2f13 100644 --- a/plugin.go +++ b/plugin.go @@ -21,18 +21,19 @@ import ( type ( // Config holds input parameters for the plugin Config struct { - Actions []string - Vars map[string]string - Secrets map[string]string - InitOptions InitOptions - FmtOptions FmtOptions - Cacert string - Sensitive bool - RoleARN string - RootDir string - Parallelism int - Targets []string - VarFiles []string + Actions []string + Vars map[string]string + Secrets map[string]string + InitOptions InitOptions + FmtOptions FmtOptions + Cacert string + Sensitive bool + RoleARN string + RootDir string + Parallelism int + Targets []string + VarFiles []string + TerraformDataDir string } // Netrc is credentials for cloning @@ -96,8 +97,8 @@ func (p Plugin) Exec() error { commands = append(commands, installCaCert(p.Config.Cacert)) } - commands = append(commands, deleteCache()) - commands = append(commands, initCommand(p.Config.InitOptions)) + commands = append(commands, deleteCache(p.Config)) + commands = append(commands, initCommand(p.Config)) commands = append(commands, getModules()) // Add commands listed from Actions @@ -120,7 +121,7 @@ func (p Plugin) Exec() error { } } - commands = append(commands, deleteCache()) + commands = append(commands, deleteCache(p.Config)) for _, c := range commands { if c.Dir == "" { @@ -183,11 +184,12 @@ func assumeRole(roleArn string) { os.Setenv("AWS_SESSION_TOKEN", value.SessionToken) } -func deleteCache() *exec.Cmd { +func deleteCache(config Config) *exec.Cmd { + terraformDataDir := getTerraformDataDir(config) return exec.Command( "rm", "-rf", - ".terraform", + terraformDataDir, ) } @@ -198,30 +200,30 @@ func getModules() *exec.Cmd { ) } -func initCommand(config InitOptions) *exec.Cmd { +func initCommand(config Config) *exec.Cmd { args := []string{ "init", } - for _, v := range config.BackendConfig { + for _, v := range config.InitOptions.BackendConfig { args = append(args, fmt.Sprintf("-backend-config=%s", v)) } // True is default in TF - if config.Lock != nil { - args = append(args, fmt.Sprintf("-lock=%t", *config.Lock)) + if config.InitOptions.Lock != nil { + args = append(args, fmt.Sprintf("-lock=%t", *config.InitOptions.Lock)) } // "0s" is default in TF - if config.LockTimeout != "" { - args = append(args, fmt.Sprintf("-lock-timeout=%s", config.LockTimeout)) + if config.InitOptions.LockTimeout != "" { + args = append(args, fmt.Sprintf("-lock-timeout=%s", config.InitOptions.LockTimeout)) } // Fail Terraform execution on prompt args = append(args, "-input=false") - return exec.Command( - "terraform", + return createTerraformCommand( + config, args..., ) } @@ -253,9 +255,10 @@ func tfApply(config Config) *exec.Cmd { if config.InitOptions.LockTimeout != "" { args = append(args, fmt.Sprintf("-lock-timeout=%s", config.InitOptions.LockTimeout)) } - args = append(args, "plan.tfout") - return exec.Command( - "terraform", + args = append(args, getTfoutPath(config)) + + return createTerraformCommand( + config, args..., ) } @@ -279,8 +282,8 @@ func tfDestroy(config Config) *exec.Cmd { args = append(args, fmt.Sprintf("-lock-timeout=%s", config.InitOptions.LockTimeout)) } args = append(args, "-force") - return exec.Command( - "terraform", + return createTerraformCommand( + config, args..., ) } @@ -293,7 +296,7 @@ func tfPlan(config Config, destroy bool) *exec.Cmd { if destroy { args = append(args, "-destroy") } else { - args = append(args, "-out=plan.tfout") + args = append(args, fmt.Sprintf("-out=%s", getTfoutPath(config))) } for _, v := range config.Targets { @@ -310,8 +313,8 @@ func tfPlan(config Config, destroy bool) *exec.Cmd { if config.InitOptions.LockTimeout != "" { args = append(args, fmt.Sprintf("-lock-timeout=%s", config.InitOptions.LockTimeout)) } - return exec.Command( - "terraform", + return createTerraformCommand( + config, args..., ) } @@ -326,8 +329,8 @@ func tfValidate(config Config) *exec.Cmd { for k, v := range config.Vars { args = append(args, "-var", fmt.Sprintf("%s=%s", k, v)) } - return exec.Command( - "terraform", + return createTerraformCommand( + config, args..., ) } @@ -348,12 +351,49 @@ func tfFmt(config Config) *exec.Cmd { if config.FmtOptions.Check != nil { args = append(args, fmt.Sprintf("-check=%t", *config.FmtOptions.Check)) } - return exec.Command( - "terraform", + return createTerraformCommand( + config, args..., ) } +func getTerraformDataDir(config Config) string { + // Override terraform data dir + var terraformDataDir string + if config.TerraformDataDir != "" { + terraformDataDir = config.TerraformDataDir + } else if os.Getenv("TF_DATA_DIR") != "" { + terraformDataDir = os.Getenv("TF_DATA_DIR") + } else { + terraformDataDir = ".terraform" + } + return terraformDataDir +} + +func createEnvironmentVariables(config Config) []string { + var environmentVariables []string = []string{} + + terraformDataDir := getTerraformDataDir(config) + if terraformDataDir != ".terraform" { + environmentVariables = append(environmentVariables, fmt.Sprintf("TF_DATA_DIR=%s", terraformDataDir)) + } + return environmentVariables +} + +func createTerraformCommand(config Config, args ...string) *exec.Cmd { + command := exec.Command("terraform", args...) + command.Env = append(command.Env, createEnvironmentVariables(config)...) + return command +} +func getTfoutPath(config Config) string { + terraformDataDir := getTerraformDataDir(config) + if terraformDataDir == ".terraform" { + return "plan.tfout" + } else { + return fmt.Sprintf("%s.plan.tfout", terraformDataDir) + } +} + func vars(vs map[string]string) []string { var args []string for k, v := range vs { diff --git a/plugin_test.go b/plugin_test.go index dc091d5..2224f4b 100644 --- a/plugin_test.go +++ b/plugin_test.go @@ -204,4 +204,59 @@ func TestPlugin(t *testing.T) { } }) }) + + g.Describe("tfDataDir", func() { + g.It("Should override the terraform data dir environment variable when provided", func() { + type args struct { + config Config + } + + tests := []struct { + name string + args args + want *exec.Cmd + expectedEnvVars []string + }{ + { + "with TerraformDataDir", + args{config: Config{TerraformDataDir: ".overriden_terraform_dir"}}, + exec.Command("terraform", "apply", ".overriden_terraform_dir.plan.tfout"), + []string{"TF_DATA_DIR=.overriden_terraform_dir"}, + }, + { + "with TerraformDataDir value as .terraform", + args{config: Config{TerraformDataDir: ".terraform"}}, + exec.Command("terraform", "apply", "plan.tfout"), + []string{}, + }, + { + "without TerraformDataDir", + args{config: Config{}}, + exec.Command("terraform", "apply", "plan.tfout"), + []string{}, + }, + } + + for _, tt := range tests { + applied := tfApply(tt.args.config) + appliedEnv := applied.Env + applied.Env = nil + + g.Assert(applied).Equal(tt.want) + + var found int = 0 + + for _, expectedEnvVar := range tt.expectedEnvVars { + for _, env := range appliedEnv { + if expectedEnvVar == env { + found += 1 + break + } + } + } + g.Assert(found).Equal(len(tt.expectedEnvVars)) + + } + }) + }) }