package build import ( "bufio" "bytes" "errors" "fmt" "go/ast" "go/parser" "go/token" "io" "log" "log/slog" "os" "os/exec" "path" "path/filepath" "regexp" "strconv" "strings" "text/template" "github.com/ldez/go-git-cmd-wrapper/v2/clone" "github.com/ldez/go-git-cmd-wrapper/v2/git" ) const ( OAS_REPO_NAME = "stackit-api-specifications" OAS_REPO = "https://github.com/stackitcloud/stackit-api-specifications.git" GEN_REPO_NAME = "stackit-sdk-generator" GEN_REPO = "https://github.com/stackitcloud/stackit-sdk-generator.git" ) type version struct { verString string major int minor int } type Builder struct { SkipClone bool SkipCleanup bool PackagesOnly bool Verbose bool Debug bool } func (b *Builder) Build() error { slog.Info("Starting Builder") if b.PackagesOnly { slog.Info(" >>> only generating pkg_gen <<<") } root, err := getRoot() if err != nil { log.Fatal(err) } if root == nil || *root == "" { return fmt.Errorf("unable to determine root directory from git") } if b.Verbose { slog.Info(" ... using root directory", "dir", *root) } if !b.PackagesOnly { if b.Verbose { slog.Info(" ... Checking needed commands available") } chkErr := checkCommands([]string{}) if chkErr != nil { return chkErr } } if !b.SkipCleanup { slog.Info("Cleaning up old packages directory") err = os.RemoveAll(path.Join(*root, "pkg_gen")) if err != nil { return err } } if !b.SkipCleanup && !b.PackagesOnly { slog.Info("Cleaning up old packages directory") err = os.RemoveAll(path.Join(*root, "pkg_gen")) if err != nil { return err } } slog.Info("Creating generator dir", "dir", fmt.Sprintf("%s/%s", *root, GEN_REPO_NAME)) genDir := path.Join(*root, GEN_REPO_NAME) if !b.SkipClone { err = createGeneratorDir(GEN_REPO, genDir, b.SkipClone) if err != nil { return err } } slog.Info("Creating oas repo dir", "dir", fmt.Sprintf("%s/%s", *root, OAS_REPO_NAME)) repoDir, err := b.createRepoDir(genDir, OAS_REPO, OAS_REPO_NAME, b.SkipClone) if err != nil { return fmt.Errorf("%s", err.Error()) } // TODO - major verMap, err := b.getVersions(repoDir) if err != nil { return fmt.Errorf("%s", err.Error()) } slog.Info("Reducing to only latest or highest") res, err := getOnlyLatest(verMap) if err != nil { return fmt.Errorf("%s", err.Error()) } slog.Info("Creating OAS dir") err = os.MkdirAll(path.Join(genDir, "oas", "legacy"), 0o755) //nolint:gosec // this dir is not sensitive, so we can use 0755 if err != nil { return err } slog.Info("Copying OAS files") for service, item := range res { baseService := strings.TrimSuffix(service, "alpha") baseService = strings.TrimSuffix(baseService, "beta") itemVersion := fmt.Sprintf("v%d%s", item.major, item.verString) if item.minor != 0 { itemVersion = itemVersion + "" + strconv.Itoa(item.minor) } srcFile := path.Join( repoDir, "services", baseService, itemVersion, fmt.Sprintf("%s.json", baseService), ) dstFile := path.Join(genDir, "oas", "legacy", fmt.Sprintf("%s.json", service)) _, err = copyFile(srcFile, dstFile) if err != nil { return fmt.Errorf("%s", err.Error()) } } slog.Info("Changing dir", "dir", genDir) err = os.Chdir(genDir) if err != nil { return err } slog.Info("Calling make", "command", "generate-go-sdk") cmd := exec.Command("make", "generate-go-sdk") var stdOut, stdErr bytes.Buffer cmd.Stdout = &stdOut cmd.Stderr = &stdErr if err = cmd.Start(); err != nil { slog.Error("cmd.Start", "error", err) return err } if err = cmd.Wait(); err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) { slog.Error( "cmd.Wait", "code", exitErr.ExitCode(), "error", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return fmt.Errorf("%s", stdErr.String()) } if err != nil { slog.Error("cmd.Wait", "err", err) return err } } slog.Info("Cleaning up go.mod and go.sum files") cleanDir := path.Join(genDir, "sdk-repo-updated", "services") dirEntries, err := os.ReadDir(cleanDir) if err != nil { return err } for _, entry := range dirEntries { if entry.IsDir() { err = deleteFiles( path.Join(cleanDir, entry.Name(), "go.mod"), path.Join(cleanDir, entry.Name(), "go.sum"), ) if err != nil { return err } } } slog.Info("Changing dir", "dir", *root) err = os.Chdir(*root) if err != nil { return err } slog.Info("Rearranging package directories") //nolint:gosec // this dir is not sensitive, so we can use 0755 err = os.MkdirAll( path.Join(*root, "pkg_gen"), 0o755, ) if err != nil { return err } srcDir := path.Join(genDir, "sdk-repo-updated", "services") items, err := os.ReadDir(srcDir) if err != nil { return err } for _, item := range items { if !item.IsDir() { continue } slog.Info(" -> package", "name", item.Name()) tgtDir := path.Join(*root, "pkg_gen", item.Name()) if fileExists(tgtDir) { delErr := os.RemoveAll(tgtDir) if delErr != nil { return delErr } } err = os.Rename(path.Join(srcDir, item.Name()), tgtDir) if err != nil { return err } } if !b.PackagesOnly { slog.Info("Generating service boilerplate") err = generateServiceFiles(*root, path.Join(*root, GEN_REPO_NAME)) if err != nil { return err } slog.Info("Copying all service files") err = CopyDirectory( path.Join(*root, "generated", "internal", "services"), path.Join(*root, "stackit", "internal", "services"), ) if err != nil { return err } err = createBoilerplate(*root, path.Join(*root, "stackit", "internal", "services")) if err != nil { return err } } if !b.SkipCleanup { slog.Info("Finally removing temporary files and directories") err = os.RemoveAll(path.Join(*root, "generated")) if err != nil { slog.Error("RemoveAll", "dir", path.Join(*root, "generated"), "err", err) return err } err = os.RemoveAll(path.Join(*root, GEN_REPO_NAME)) if err != nil { slog.Error("RemoveAll", "dir", path.Join(*root, GEN_REPO_NAME), "err", err) return err } slog.Info("Cleaning up", "dir", repoDir) err = os.RemoveAll(filepath.Dir(repoDir)) if err != nil { return fmt.Errorf("%s", err.Error()) } } slog.Info("Done") return nil } type templateData struct { PackageName string PackageNameCamel string PackageNamePascal string NameCamel string NamePascal string NameSnake string Fields []string } func fileExists(pathValue string) bool { _, err := os.Stat(pathValue) if os.IsNotExist(err) { return false } if err != nil { panic(err) } return true } func createBoilerplate(rootFolder, folder string) error { services, err := os.ReadDir(folder) if err != nil { return err } for _, svc := range services { if !svc.IsDir() { continue } resources, err := os.ReadDir(path.Join(folder, svc.Name())) if err != nil { return err } var handleDS bool var handleRes bool var foundDS bool var foundRes bool for _, res := range resources { if !res.IsDir() { continue } resourceName := res.Name() dsFile := path.Join( folder, svc.Name(), res.Name(), "datasources_gen", fmt.Sprintf("%s_data_source_gen.go", res.Name()), ) handleDS = fileExists(dsFile) resFile := path.Join( folder, svc.Name(), res.Name(), "resources_gen", fmt.Sprintf("%s_resource_gen.go", res.Name()), ) handleRes = fileExists(resFile) dsGoFile := path.Join(folder, svc.Name(), res.Name(), "datasource.go") foundDS = fileExists(dsGoFile) resGoFile := path.Join(folder, svc.Name(), res.Name(), "resource.go") foundRes = fileExists(resGoFile) if handleDS && !foundDS { slog.Info(" creating missing datasource.go", "service", svc.Name(), "resource", resourceName) if !ValidateSnakeCase(resourceName) { return errors.New("resource name is invalid") } fields, tokenErr := getTokens(dsFile) if tokenErr != nil { return fmt.Errorf("error reading tokens: %w", tokenErr) } tplName := "data_source_scaffold.gotmpl" err = writeTemplateToFile( tplName, path.Join(rootFolder, "cmd", "cmd", "build", "templates", tplName), dsGoFile, &templateData{ PackageName: svc.Name(), PackageNameCamel: ToCamelCase(svc.Name()), PackageNamePascal: ToPascalCase(svc.Name()), NameCamel: ToCamelCase(resourceName), NamePascal: ToPascalCase(resourceName), NameSnake: resourceName, Fields: fields, }, ) if err != nil { panic(err) } } if handleRes && !foundRes { slog.Info(" creating missing resource.go", "service", svc.Name(), "resource", resourceName) if !ValidateSnakeCase(resourceName) { return errors.New("resource name is invalid") } fields, tokenErr := getTokens(resFile) if tokenErr != nil { return fmt.Errorf("error reading tokens: %w", tokenErr) } tplName := "resource_scaffold.gotmpl" err = writeTemplateToFile( tplName, path.Join(rootFolder, "cmd", "cmd", "build", "templates", tplName), resGoFile, &templateData{ PackageName: svc.Name(), PackageNameCamel: ToCamelCase(svc.Name()), PackageNamePascal: ToPascalCase(svc.Name()), NameCamel: ToCamelCase(resourceName), NamePascal: ToPascalCase(resourceName), NameSnake: resourceName, Fields: fields, }, ) if err != nil { return err } if !fileExists(path.Join(folder, svc.Name(), res.Name(), "functions.go")) { slog.Info(" creating missing functions.go", "service", svc.Name(), "resource", resourceName) if !ValidateSnakeCase(resourceName) { return errors.New("resource name is invalid") } fncTplName := "functions_scaffold.gotmpl" err = writeTemplateToFile( fncTplName, path.Join(rootFolder, "cmd", "cmd", "build", "templates", fncTplName), path.Join(folder, svc.Name(), res.Name(), "functions.go"), &templateData{ PackageName: svc.Name(), PackageNameCamel: ToCamelCase(svc.Name()), PackageNamePascal: ToPascalCase(svc.Name()), NameCamel: ToCamelCase(resourceName), NamePascal: ToPascalCase(resourceName), NameSnake: resourceName, }, ) if err != nil { return err } } } } } return nil } func ucfirst(s string) string { if s == "" { return "" } return strings.ToUpper(s[:1]) + s[1:] } func writeTemplateToFile(tplName, tplFile, outFile string, data *templateData) error { fn := template.FuncMap{ "ucfirst": ucfirst, } tmpl, err := template.New(tplName).Funcs(fn).ParseFiles(tplFile) if err != nil { return err } var f *os.File f, err = os.Create(outFile) if err != nil { return err } err = tmpl.Execute(f, *data) if err != nil { return err } err = f.Close() if err != nil { return err } return nil } func generateServiceFiles(rootDir, generatorDir string) error { //nolint:gosec // this file is not sensitive, so we can use 0755 err := os.MkdirAll(path.Join(rootDir, "generated", "specs"), 0o755) if err != nil { return err } services, err := os.ReadDir(path.Join(rootDir, "service_specs")) if err != nil { return err } for _, service := range services { if !service.IsDir() { continue } versions, err := os.ReadDir(path.Join(rootDir, "service_specs", service.Name())) if err != nil { return err } for _, svcVersion := range versions { if !svcVersion.IsDir() { continue } // TODO: use const of supported versions if svcVersion.Name() != "alpha" && svcVersion.Name() != "beta" { continue } specFiles, err := os.ReadDir(path.Join(rootDir, "service_specs", service.Name(), svcVersion.Name())) if err != nil { return err } for _, specFile := range specFiles { if specFile.IsDir() { continue } r := regexp.MustCompile(`^(.*)_config.yml$`) matches := r.FindAllStringSubmatch(specFile.Name(), -1) if matches != nil { fileName := matches[0][0] resource := matches[0][1] slog.Info( " found service spec", "name", specFile.Name(), "service", service.Name(), "resource", resource, ) oasFile := path.Join( generatorDir, "oas", "legacy", fmt.Sprintf("%s%s.json", service.Name(), svcVersion.Name()), ) if _, oasErr := os.Stat(oasFile); os.IsNotExist(oasErr) { slog.Warn( " could not find matching oas", "svc", service.Name(), "version", svcVersion.Name(), ) continue } scName := fmt.Sprintf("%s%s", service.Name(), svcVersion.Name()) scName = strings.ReplaceAll(scName, "-", "") //nolint:gosec // this file is not sensitive, so we can use 0755 err = os.MkdirAll(path.Join(rootDir, "generated", "internal", "services", scName, resource), 0o755) if err != nil { return err } specJsonFile := path.Join( rootDir, "generated", "specs", fmt.Sprintf("%s_%s_spec.json", scName, resource), ) var stdOut, stdErr bytes.Buffer // nolint:gosec // #nosec this command is not using any untrusted input, so we can ignore gosec warning cmd := exec.Command( "go", "run", "github.com/hashicorp/terraform-plugin-codegen-openapi/cmd/tfplugingen-openapi", "generate", "--config", path.Join(rootDir, "service_specs", service.Name(), svcVersion.Name(), fileName), "--output", specJsonFile, oasFile, ) cmd.Stdout = &stdOut cmd.Stderr = &stdErr if err = cmd.Start(); err != nil { slog.Error( "tfplugingen-openapi generate", "error", err, "stdOut", stdOut.String(), "stdErr", stdErr.String(), ) return err } if err = cmd.Wait(); err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) { slog.Error( "tfplugingen-openapi generate", "code", exitErr.ExitCode(), "error", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return fmt.Errorf("%s", stdErr.String()) } if err != nil { slog.Error( "tfplugingen-openapi generate", "err", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return err } } if stdOut.Len() > 0 { slog.Warn(" command output", "stdout", stdOut.String(), "stderr", stdErr.String()) } tgtFolder := path.Join( rootDir, "generated", "internal", "services", scName, resource, "resources_gen", ) //nolint:gosec // this file is not sensitive, so we can use 0755 err = os.MkdirAll(tgtFolder, 0o755) if err != nil { return err } // nolint:gosec // #nosec this command is not using any untrusted input, so we can ignore gosec warning cmd2 := exec.Command( "go", "run", "github.com/hashicorp/terraform-plugin-codegen-framework/cmd/tfplugingen-framework", "generate", "resources", "--input", specJsonFile, "--output", tgtFolder, "--package", scName, ) cmd2.Stdout = &stdOut cmd2.Stderr = &stdErr if err = cmd2.Start(); err != nil { slog.Error("tfplugingen-framework generate resources", "error", err) return err } if err = cmd2.Wait(); err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) { slog.Error( "tfplugingen-framework generate resources", "code", exitErr.ExitCode(), "error", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return fmt.Errorf("%s", stdErr.String()) } if err != nil { slog.Error( "tfplugingen-framework generate resources", "err", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return err } } tgtFolder = path.Join( rootDir, "generated", "internal", "services", scName, resource, "datasources_gen", ) //nolint:gosec // this directory is not sensitive, so we can use 0755 err = os.MkdirAll(tgtFolder, 0o755) if err != nil { return err } // nolint:gosec // #nosec this command is not using any untrusted input, so we can ignore gosec warning cmd3 := exec.Command( "go", "run", "github.com/hashicorp/terraform-plugin-codegen-framework/cmd/tfplugingen-framework", "generate", "data-sources", "--input", specJsonFile, "--output", tgtFolder, "--package", scName, ) var stdOut3, stdErr3 bytes.Buffer cmd3.Stdout = &stdOut3 cmd3.Stderr = &stdErr3 if err = cmd3.Start(); err != nil { slog.Error("tfplugingen-framework generate data-sources", "error", err) return err } if err = cmd3.Wait(); err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) { slog.Error( "tfplugingen-framework generate data-sources", "code", exitErr.ExitCode(), "error", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return fmt.Errorf("%s", stdErr.String()) } if err != nil { slog.Error( "tfplugingen-framework generate data-sources", "err", err, "stdout", stdOut.String(), "stderr", stdErr.String(), ) return err } } tfAnoErr := handleTfTagForDatasourceFile( path.Join(tgtFolder, fmt.Sprintf("%s_data_source_gen.go", resource)), scName, resource, ) if tfAnoErr != nil { return tfAnoErr } } } } } return nil } // handleTfTagForDatasourceFile replaces existing "id" with "stf_original_api_id" func handleTfTagForDatasourceFile(filePath, service, resource string) error { slog.Info(" handle terraform tag for datasource", "service", service, "resource", resource) if !fileExists(filePath) { slog.Warn(" could not find file, skipping", "path", filePath) return nil } f, err := os.Open(filePath) if err != nil { return err } root, err := getRoot() if err != nil { //nolint:gocritic // in this case, we want to log the error and exit, as we cannot proceed without the root directory log.Fatal(err) } tmp, err := os.CreateTemp(*root, "replace-*") if err != nil { return err } sc := bufio.NewScanner(f) for sc.Scan() { resLine, err := handleLine(sc.Text()) if err != nil { return err } if _, err := tmp.WriteString(resLine + "\n"); err != nil { return err } } if scErr := sc.Err(); scErr != nil { return scErr } if err := tmp.Close(); err != nil { return err } if err := f.Close(); err != nil { return err } //nolint:gosec // path traversal is not a concern here if err := os.Rename(tmp.Name(), filePath); err != nil { log.Fatal(err) } return nil } func handleLine(line string) (string, error) { schemaRegex := regexp.MustCompile(`(\s+")(id)(": schema.[a-zA-Z0-9]+Attribute{)`) schemaMatches := schemaRegex.FindAllStringSubmatch(line, -1) if schemaMatches != nil { return fmt.Sprintf("%stf_original_api_id%s", schemaMatches[0][1], schemaMatches[0][3]), nil } modelRegex := regexp.MustCompile(`(\s+Id\s+types.[a-zA-Z0-9]+\s+.tfsdk:")(id)(".)`) modelMatches := modelRegex.FindAllStringSubmatch(line, -1) if modelMatches != nil { return fmt.Sprintf("%stf_original_api_id%s", modelMatches[0][1], modelMatches[0][3]), nil } return line, nil } func checkCommands(commands []string) error { for _, commandName := range commands { if !commandExists(commandName) { return fmt.Errorf("missing command %s", commandName) } slog.Info(" found", "command", commandName) } return nil } func commandExists(cmd string) bool { _, err := exec.LookPath(cmd) return err == nil } func deleteFiles(fNames ...string) error { for _, fName := range fNames { if _, err := os.Stat(fName); !os.IsNotExist(err) { err = os.Remove(fName) if err != nil { return err } } } return nil } func copyFile(src, dst string) (int64, error) { sourceFileStat, err := os.Stat(src) if err != nil { return 0, err } if !sourceFileStat.Mode().IsRegular() { return 0, fmt.Errorf("%s is not a regular file", src) } source, err := os.Open(src) if err != nil { return 0, err } defer func(source *os.File) { err := source.Close() if err != nil { slog.Error("copyFile", "err", err) } }(source) destination, err := os.Create(dst) if err != nil { return 0, err } defer func(destination *os.File) { err := destination.Close() if err != nil { slog.Error("copyFile", "err", err) } }(destination) nBytes, err := io.Copy(destination, source) return nBytes, err } func getOnlyLatest(m map[string]version) (map[string]version, error) { tmpMap := make(map[string]version) for k, v := range m { item, ok := tmpMap[k] if !ok { tmpMap[k] = v } else if item.major == v.major && item.minor < v.minor { tmpMap[k] = v } } return tmpMap, nil } func (b *Builder) getVersions(dir string) (map[string]version, error) { slog.Info("Retrieving versions from subdirs", "func", "getVersions") res := make(map[string]version) children, err := os.ReadDir(path.Join(dir, "services")) if err != nil { return nil, err } if len(children) < 1 { slog.Error("found no children", "dir", path.Join(dir, "services")) } for _, entry := range children { if !entry.IsDir() { slog.Info("entry is no dir", "entry", entry.Name()) continue } if b.Verbose { slog.Info("getting versions", "svc", entry.Name()) } versions, err := os.ReadDir(path.Join(dir, "services", entry.Name())) if err != nil { return nil, err } m, err2 := b.extractVersions(entry.Name(), versions) if err2 != nil { return m, err2 } for k, v := range m { res[k] = v } } return res, nil } func (b *Builder) extractVersions(service string, versionDirs []os.DirEntry) (map[string]version, error) { res := make(map[string]version) if len(versionDirs) < 1 { slog.Error("list of version directories is empty") return nil, nil } for _, vDir := range versionDirs { if !vDir.IsDir() { continue } r := regexp.MustCompile(`v(\d+)([a-z]+)(\d*)`) matches := r.FindAllStringSubmatch(vDir.Name(), -1) if matches == nil { if b.Debug { slog.Warn("item did not fulfill regex", "item", vDir.Name()) } continue } svc, ver, err := handleVersion(service, matches[0]) if err != nil { return nil, err } if svc != nil && ver != nil { res[*svc] = *ver } } return res, nil } func handleVersion(service string, match []string) (*string, *version, error) { if match == nil { fmt.Println("no matches") return nil, nil, nil } verString := match[2] if verString != "alpha" && verString != "beta" { return nil, nil, errors.New("unsupported version") } majVer, err := strconv.Atoi(match[1]) if err != nil { return nil, nil, err } if match[3] == "" { match[3] = "0" } minVer, err := strconv.Atoi(match[3]) if err != nil { return nil, nil, err } resStr := fmt.Sprintf("%s%s", service, verString) return &resStr, &version{verString: verString, major: majVer, minor: minVer}, nil } func (b *Builder) createRepoDir(root, repoUrl, repoName string, skipClone bool) (string, error) { targetDir := path.Join(root, repoName) if !skipClone { if fileExists(targetDir) { slog.Warn("target dir exists - skipping", "targetDir", targetDir) return targetDir, nil } out, err := git.Clone( clone.Repository(repoUrl), clone.Directory(targetDir), ) if err != nil { return "", err } if b.Verbose { slog.Info("git clone result", "output", out) } } return targetDir, nil } func createGeneratorDir(repoUrl, targetDir string, skipClone bool) error { if !skipClone { if fileExists(targetDir) { remErr := os.RemoveAll(targetDir) if remErr != nil { return remErr } } _, cloneErr := git.Clone( clone.Repository(repoUrl), clone.Directory(targetDir), ) if cloneErr != nil { return cloneErr } } return nil } func getRoot() (*string, error) { cmd := exec.Command("git", "rev-parse", "--show-toplevel") out, err := cmd.Output() if err != nil { return nil, err } lines := strings.Split(string(out), "\n") return &lines[0], nil } func getTokens(fileName string) ([]string, error) { fset := token.NewFileSet() var result []string node, err := parser.ParseFile(fset, fileName, nil, parser.ParseComments) if err != nil { return nil, err } ast.Inspect( node, func(n ast.Node) bool { // Suche nach Typ-Deklarationen (structs) ts, ok := n.(*ast.TypeSpec) if ok { if strings.Contains(ts.Name.Name, "Model") { ast.Inspect( ts, func(sn ast.Node) bool { tts, tok := sn.(*ast.Field) if tok { result = append(result, tts.Names[0].String()) } return true }, ) } } return true }, ) return result, nil }