chore: work save

generator/cmd/build/functions.go

@@ -0,0 +1,119 @@
+package build
+
+import (
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"strings"
+	"text/template"
+)
+
+func FileExists(pathValue string) bool {
+	_, err := os.Stat(pathValue)
+	if os.IsNotExist(err) {
+		return false
+	}
+	if err != nil {
+		panic(err)
+	}
+	return true
+}
+
+func ucfirst(s string) string {
+	if s == "" {
+		return ""
+	}
+	return strings.ToUpper(s[:1]) + s[1:]
+}
+
+func writeTemplateToFile(tplName, tplFile, outFile string, data *templateData) error {
+	fn := template.FuncMap{
+		"ucfirst": ucfirst,
+	}
+
+	tmpl, err := template.New(tplName).Funcs(fn).ParseFiles(tplFile)
+	if err != nil {
+		return err
+	}
+
+	var f *os.File
+	f, err = os.Create(outFile)
+	if err != nil {
+		return err
+	}
+
+	err = tmpl.Execute(f, *data)
+	if err != nil {
+		return err
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func deleteFiles(fNames ...string) error {
+	for _, fName := range fNames {
+		if _, err := os.Stat(fName); !os.IsNotExist(err) {
+			err = os.Remove(fName)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func copyFile(src, dst string) (int64, error) {
+	sourceFileStat, err := os.Stat(src)
+	if err != nil {
+		return 0, err
+	}
+
+	if !sourceFileStat.Mode().IsRegular() {
+		return 0, fmt.Errorf("%s is not a regular file", src)
+	}
+
+	source, err := os.Open(src)
+	if err != nil {
+		return 0, err
+	}
+	defer func(source *os.File) {
+		err := source.Close()
+		if err != nil {
+			slog.Error("copyFile", "err", err)
+		}
+	}(source)
+
+	destination, err := os.Create(dst)
+	if err != nil {
+		return 0, err
+	}
+	defer func(destination *os.File) {
+		err := destination.Close()
+		if err != nil {
+			slog.Error("copyFile", "err", err)
+		}
+	}(destination)
+	nBytes, err := io.Copy(destination, source)
+	return nBytes, err
+}
+
+func checkCommands(commands []string) error {
+	for _, commandName := range commands {
+		if !commandExists(commandName) {
+			return fmt.Errorf("missing command %s", commandName)
+		}
+		slog.Info("        found", "command", commandName)
+	}
+	return nil
+}
+
+func commandExists(cmd string) bool {
+	_, err := exec.LookPath(cmd)
+	return err == nil
+}

generator/cmd/build/oas-handler.go

@@ -0,0 +1,446 @@
+package build
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"log"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/ldez/go-git-cmd-wrapper/v2/clone"
+	"github.com/ldez/go-git-cmd-wrapper/v2/git"
+)
+
+const (
+	OasRepoName = "stackit-api-specifications"
+	OasRepo     = "https://github.com/stackitcloud/stackit-api-specifications.git"
+
+	ResTypeResource   = "resources"
+	ResTypeDataSource = "datasources"
+)
+
+type Data struct {
+	ServiceName string    `yaml:",omitempty" json:",omitempty"`
+	Versions    []Version `yaml:"versions" json:"versions"`
+}
+
+type Version struct {
+	Name string `yaml:"name" json:"name"`
+	Path string `yaml:"path" json:"path"`
+}
+
+var oasTempDir string
+
+func (b *Builder) oasHandler(specDir string) error {
+	if b.Verbose {
+		slog.Info("creating schema files", "dir", specDir)
+	}
+	if _, err := os.Stat(specDir); os.IsNotExist(err) {
+		return fmt.Errorf("spec files directory does not exist")
+	}
+
+	err := b.createRepoDir(b.SkipClone)
+	if err != nil {
+		return fmt.Errorf("%s", err.Error())
+	}
+
+	err2 := b.handleServices(specDir)
+	if err2 != nil {
+		return err2
+	}
+
+	if !b.SkipCleanup {
+		if b.Verbose {
+			slog.Info("Finally removing temporary files and directories")
+		}
+		err := os.RemoveAll(path.Join(b.rootDir, "generated"))
+		if err != nil {
+			slog.Error("RemoveAll", "dir", path.Join(b.rootDir, "generated"), "err", err)
+			return err
+		}
+
+		err = os.RemoveAll(oasTempDir)
+		if err != nil {
+			slog.Error("RemoveAll", "dir", oasTempDir, "err", err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *Builder) handleServices(specDir string) error {
+	services, err := os.ReadDir(specDir)
+	if err != nil {
+		return err
+	}
+
+	for _, svc := range services {
+		if !svc.IsDir() {
+			continue
+		}
+
+		if b.Verbose {
+			slog.Info("  ... found", "service", svc.Name())
+		}
+		var svcVersions Data
+		svcVersions.ServiceName = svc.Name()
+
+		versionsErr := b.getServiceVersions(path.Join(specDir, svc.Name(), "generator_settings.yml"), &svcVersions)
+		if versionsErr != nil {
+			return versionsErr
+		}
+
+		oasSpecErr := b.generateServiceFiles(&svcVersions)
+		if oasSpecErr != nil {
+			return oasSpecErr
+		}
+	}
+	return nil
+}
+
+func (b *Builder) getServiceVersions(confFile string, data *Data) error {
+	if _, cfgFileErr := os.Stat(confFile); os.IsNotExist(cfgFileErr) {
+		return fmt.Errorf("config file does not exist")
+	}
+
+	fileContent, fileErr := os.ReadFile(confFile)
+	if fileErr != nil {
+		return fileErr
+	}
+	convErr := yaml.Unmarshal(fileContent, &data)
+	if convErr != nil {
+		return convErr
+	}
+
+	return nil
+}
+
+func (b *Builder) createRepoDir(skipClone bool) error {
+	tmpDirName, err := os.MkdirTemp("", "oasbuild")
+	if err != nil {
+		return err
+	}
+	oasTempDir = path.Join(tmpDirName, OasRepoName)
+	slog.Info("Creating oas repo dir", "dir", oasTempDir)
+	if !skipClone {
+		if FileExists(oasTempDir) {
+			slog.Warn("target dir exists - skipping", "targetDir", oasTempDir)
+			return nil
+		}
+		out, cloneErr := git.Clone(
+			clone.Repository(OasRepo),
+			clone.Directory(oasTempDir),
+		)
+		if cloneErr != nil {
+			slog.Error("git clone error", "output", out)
+			return cloneErr
+		}
+		if b.Verbose {
+			slog.Info("git clone result", "output", out)
+		}
+	}
+	return nil
+}
+
+func (b *Builder) generateServiceFiles(data *Data) error {
+	err := os.MkdirAll(path.Join(b.rootDir, "generated", "specs"), 0o750)
+	if err != nil {
+		return err
+	}
+
+	for _, v := range data.Versions {
+		specFiles, specsErr := os.ReadDir(path.Join(b.rootDir, "service_specs", data.ServiceName, v.Name))
+		if specsErr != nil {
+			return specsErr
+		}
+		for _, specFile := range specFiles {
+			if specFile.IsDir() {
+				continue
+			}
+			r := regexp.MustCompile(`^(.*)_config.yml$`)
+			matches := r.FindAllStringSubmatch(specFile.Name(), -1)
+			if matches == nil {
+				slog.Warn("    skipping file (no regex match)", "file", specFile.Name())
+				continue
+			}
+
+			srcSpecFile := path.Join(b.rootDir, "service_specs", data.ServiceName, v.Name, specFile.Name())
+
+			if matches[0][0] != specFile.Name() {
+				return fmt.Errorf("matched filename differs from original filename - this should not happen")
+			}
+			resource := matches[0][1]
+			if b.Verbose {
+				slog.Info(
+					"    found service spec",
+					"service",
+					data.ServiceName,
+					"resource",
+					resource,
+					"file",
+					specFile.Name(),
+				)
+			}
+
+			oasFile := path.Join(
+				oasTempDir,
+				"services",
+				data.ServiceName,
+				v.Path,
+				fmt.Sprintf("%s.json", data.ServiceName),
+			)
+			if _, oasErr := os.Stat(oasFile); os.IsNotExist(oasErr) {
+				slog.Warn(
+					"        could not find matching oas",
+					"svc",
+					data.ServiceName,
+					"version",
+					v.Name,
+				)
+				continue
+			}
+
+			// determine correct target service name
+			scName := fmt.Sprintf("%s%s", data.ServiceName, v.Name)
+			scName = strings.ReplaceAll(scName, "-", "")
+
+			specJSONFile := path.Join(
+				b.rootDir,
+				"generated",
+				"specs",
+				fmt.Sprintf("%s_%s_spec.json", scName, resource),
+			)
+
+			cmdErr := b.runTerraformPluginGenOpenAPI(srcSpecFile, specJSONFile, oasFile)
+			if cmdErr != nil {
+				return cmdErr
+			}
+
+			cmdResGenErr := b.runTerraformPluginGenFramework(ResTypeResource, scName, resource, specJSONFile)
+			if cmdResGenErr != nil {
+				return cmdResGenErr
+			}
+
+			cmdDsGenErr := b.runTerraformPluginGenFramework(ResTypeDataSource, scName, resource, specJSONFile)
+			if cmdDsGenErr != nil {
+				return cmdDsGenErr
+			}
+		}
+	}
+
+	return nil
+}
+
+func (b *Builder) runTerraformPluginGenFramework(resType, svcName, resource, specJSONFile string) error {
+	var stdOut, stdErr bytes.Buffer
+	tgtFolder := path.Join(
+		b.rootDir,
+		"stackit",
+		"internal",
+		"services",
+		svcName,
+		resource,
+		fmt.Sprintf("%s_gen", resType),
+	)
+
+	//nolint:gosec // this file is not sensitive, so we can use 0755
+	err := os.MkdirAll(tgtFolder, 0o755)
+	if err != nil {
+		return err
+	}
+
+	var subCmd string
+	switch resType {
+	case ResTypeResource:
+		subCmd = "resources"
+	case ResTypeDataSource:
+		subCmd = "data-sources"
+	default:
+		return fmt.Errorf("unknown resource type given: %s", resType)
+	}
+
+	// nolint:gosec // #nosec this command is not using any untrusted input, so we can ignore gosec warning
+	cmd := exec.Command(
+		"tfplugingen-framework",
+		"generate",
+		subCmd,
+		"--input",
+		specJSONFile,
+		"--output",
+		tgtFolder,
+		"--package",
+		svcName,
+	)
+
+	cmd.Stdout = &stdOut
+	cmd.Stderr = &stdErr
+	if err = cmd.Start(); err != nil {
+		slog.Error(fmt.Sprintf("tfplugingen-framework generate %s", resType), "error", err)
+		return err
+	}
+
+	if err = cmd.Wait(); err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			slog.Error(
+				fmt.Sprintf("tfplugingen-framework generate %s", resType),
+				"code",
+				exitErr.ExitCode(),
+				"error",
+				err,
+				"stdout",
+				stdOut.String(),
+				"stderr",
+				stdErr.String(),
+			)
+			return fmt.Errorf("%s", stdErr.String())
+		}
+		if err != nil {
+			slog.Error(
+				fmt.Sprintf("tfplugingen-framework generate %s", resType),
+				"err",
+				err,
+				"stdout",
+				stdOut.String(),
+				"stderr",
+				stdErr.String(),
+			)
+			return err
+		}
+	}
+
+	if resType == ResTypeDataSource {
+		tfAnoErr := b.handleTfTagForDatasourceFile(
+			path.Join(tgtFolder, fmt.Sprintf("%s_data_source_gen.go", resource)),
+			svcName,
+			resource,
+		)
+		if tfAnoErr != nil {
+			return tfAnoErr
+		}
+	}
+
+	return nil
+}
+
+func (b *Builder) runTerraformPluginGenOpenAPI(srcSpecFile, specJSONFile, oasFile string) error {
+	var stdOut, stdErr bytes.Buffer
+
+	// nolint:gosec // #nosec this command is not using any untrusted input, so we can ignore gosec warning
+	cmd := exec.Command(
+		"tfplugingen-openapi",
+		"generate",
+		"--config",
+		srcSpecFile,
+		"--output",
+		specJSONFile,
+		oasFile,
+	)
+	cmd.Stdout = &stdOut
+	cmd.Stderr = &stdErr
+
+	if err := cmd.Start(); err != nil {
+		slog.Error(
+			"tfplugingen-openapi generate",
+			"error",
+			err,
+			"stdOut",
+			stdOut.String(),
+			"stdErr",
+			stdErr.String(),
+		)
+		return err
+	}
+
+	if err := cmd.Wait(); err != nil {
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			slog.Error(
+				"tfplugingen-openapi generate",
+				"code",
+				exitErr.ExitCode(),
+				"error",
+				err,
+				"stdout",
+				stdOut.String(),
+				"stderr",
+				stdErr.String(),
+			)
+			return fmt.Errorf("%s", stdErr.String())
+		}
+		if err != nil {
+			slog.Error(
+				"tfplugingen-openapi generate",
+				"err",
+				err,
+				"stdout",
+				stdOut.String(),
+				"stderr",
+				stdErr.String(),
+			)
+			return err
+		}
+	}
+	if stdOut.Len() > 0 {
+		slog.Warn("    command output", "stdout", stdOut.String(), "stderr", stdErr.String())
+	}
+
+	return nil
+}
+
+// handleTfTagForDatasourceFile replaces existing "id" with "stf_original_api_id"
+func (b *Builder) handleTfTagForDatasourceFile(filePath, service, resource string) error {
+	if b.Verbose {
+		slog.Info("   handle terraform tag for datasource", "service", service, "resource", resource)
+	}
+	if !FileExists(filePath) {
+		slog.Warn("        could not find file, skipping", "path", filePath)
+		return nil
+	}
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+
+	tmp, err := os.CreateTemp(b.rootDir, "replace-*")
+	if err != nil {
+		return err
+	}
+
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		resLine, err := handleLine(sc.Text())
+		if err != nil {
+			return err
+		}
+		if _, err := tmp.WriteString(resLine + "\n"); err != nil {
+			return err
+		}
+	}
+	if scErr := sc.Err(); scErr != nil {
+		return scErr
+	}
+
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+
+	if err := f.Close(); err != nil {
+		return err
+	}
+
+	//nolint:gosec // path traversal is not a concern here
+	if err := os.Rename(tmp.Name(), filePath); err != nil {
+		log.Fatal(err)
+	}
+	return nil
+}