Improved AMI import feature, WIP

INSTALL.md

@@ -138,7 +138,33 @@ Validate the new key is available
 }
 ```
 
-## Create AMI template
+## Create AMI template (new method)
+
+Use the hive CLI tool to import a selected OS image. Note, the source file can be compressed (e.g image.tar.gz, image.gz, image.tar.xz) and the tool will automatically extract and upload the raw OS image as an AMI.
+
+Note, when downloading OS images, use supported platforms that support the `cloud-init` feature to automatically bootstrap when using the Hive EC2 functionality.
+
+Example, download the Debian 12 image from the repository [https://cloud.debian.org/images/cloud/bookworm/latest/](https://cloud.debian.org/images/cloud/bookworm/latest/)
+
+Download the image:
+
+```
+wget https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.tar.xz -O ~/debian-12-genericcloud-arm64.tar.xz
+```
+
+Import as an AMI to the backend store:
+
+```
+./bin/hive admin images import --file ~/debian-12-genericcloud-arm64.tar.xz --arch aarch64 --distro debian --version 12
+```
+
+Next, verify available disk images to confirm the import was successful
+
+```
+aws ec2 describe-images
+```
+
+## Create AMI template (old method)
 
 ### Step 1
 

cmd/hive/cmd/admin.go

@@ -32,7 +32,11 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/mulgadc/hive/hive/config"
+	"github.com/mulgadc/hive/hive/utils"
 	"github.com/mulgadc/viperblock/viperblock"
+	"github.com/mulgadc/viperblock/viperblock/backends/s3"
+	vutils "github.com/mulgadc/viperblock/viperblock/utils"
 	"github.com/spf13/cobra"
 	"gopkg.in/ini.v1"
 )
@@ -117,6 +121,15 @@ func init() {
 
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
 
+	cfgFile, _ := cmd.Flags().GetString("config")
+
+	if cfgFile == "" {
+
+		homeDir, _ := os.UserHomeDir()
+		cfgFile = fmt.Sprintf("%s/hive/config/hive.toml", homeDir)
+
+	}
+
 	imageFile, _ := cmd.Flags().GetString("file")
 
 	if imageFile == "" {
@@ -183,7 +196,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	manifest := viperblock.VolumeConfig{}
 
 	// Calculate the size
-	manifest.AMIMetadata.Name = fmt.Sprintf("%s-%s-%s-ami", distro, version, arch)
+	manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", distro, version, arch)
 
 	manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Hive", manifest.AMIMetadata.Name)
 	manifest.AMIMetadata.Architecture = arch
@@ -222,6 +235,60 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 
 	// Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image
 
+	tmpDir, err := os.MkdirTemp("", "hive-image-tmp-*")
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not create temp dir: %v\n", err)
+		os.Exit(1)
+	}
+
+	imagePath, err = utils.ExtractDiskImageFromFile(imageFile, tmpDir)
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not extract image: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Upload the image to S3 (predastore)
+
+	appConfig, err := config.LoadConfig(cfgFile)
+
+	if err != nil {
+		fmt.Println("Error loading config file:", err)
+		return
+	}
+
+	s3Config := s3.S3Config{
+		VolumeName: manifest.AMIMetadata.Name,
+		VolumeSize: uint64(imageStat.Size()),
+		Bucket:     appConfig.Predastore.Bucket,
+		Region:     appConfig.Predastore.Region,
+		AccessKey:  appConfig.AccessKey,
+		SecretKey:  appConfig.SecretKey,
+		Host:       appConfig.Predastore.Host,
+	}
+
+	vbConfig := viperblock.VB{
+		VolumeName: manifest.AMIMetadata.Name,
+		VolumeSize: uint64(imageStat.Size()),
+		BaseDir:    tmpDir,
+		Cache: viperblock.Cache{
+			Config: viperblock.CacheConfig{
+				Size: 0,
+			},
+		},
+		VolumeConfig: manifest,
+	}
+
+	err = vutils.ImportDiskImage(&s3Config, &vbConfig, imagePath)
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err)
+		os.Exit(1)
+	}
+
+	defer os.RemoveAll(tmpDir)
+
 }
 
 func runAdminInit(cmd *cobra.Command, args []string) {

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/kdomanski/iso9660 v0.4.0
 	github.com/mulgadc/predastore v0.1.5
-	github.com/mulgadc/viperblock v0.1.0
+	github.com/mulgadc/viperblock v0.1.1
 	github.com/nats-io/nats-server/v2 v2.11.6
 	github.com/nats-io/nats.go v1.43.0
 	github.com/spf13/cobra v1.9.1

go.sum

@@ -51,6 +51,8 @@ github.com/mulgadc/predastore v0.1.5 h1:7SySkQQ5MQ1NTcKcB14pCUIJipq0YVgCJZeKM6yt
 github.com/mulgadc/predastore v0.1.5/go.mod h1:BhMymRV3zMTu2S+5pJnNDDm5+AGakjhSc/PKX0q8Xfg=
 github.com/mulgadc/viperblock v0.1.0 h1:vReunbVOrxzlw7n+GI35lDRHdWBR0HeslZun7t7I0Kk=
 github.com/mulgadc/viperblock v0.1.0/go.mod h1:LPXBqDzIVQFs8VvVkTgAiZ4vQhe079hvB77mYzN6+/g=
+github.com/mulgadc/viperblock v0.1.1 h1:R1DVX1S+/xPPMQ4LIFkwIX7Es+RS3Wgdd8bsWLTEvgY=
+github.com/mulgadc/viperblock v0.1.1/go.mod h1:zCjdQ76fi2Qs2a5ign9A/Uqgbz77xts5WV0OR1be48I=
 github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI=
 github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
 github.com/nats-io/nats-server/v2 v2.11.6 h1:4VXRjbTUFKEB+7UoaKL3F5Y83xC7MxPoIONOnGgpkHw=

hive/utils/utils.go

@@ -3,11 +3,13 @@ package utils
 import (
 	"bufio"
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
 	"log/slog"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -18,8 +20,12 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/hive/hive/config"
 )
 
 func ReadPidFile(name string) (int, error) {
@@ -486,3 +492,23 @@ func validateDiskImagePath(diskimage string) (err error) {
 	return errors.New("no valid disk image found")
 
 }
+
+func CreateS3Client(cfg *config.Config) *s3.S3 {
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	httpClient := &http.Client{Transport: tr}
+
+	sess := session.Must(session.NewSession(&aws.Config{
+		Region:           aws.String(cfg.Predastore.Region),
+		Endpoint:         aws.String(fmt.Sprintf("https://%s", cfg.Predastore.Host)),
+		Credentials:      credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, ""),
+		S3ForcePathStyle: aws.Bool(true),
+		DisableSSL:       aws.Bool(false),
+		HTTPClient:       httpClient,
+	}))
+
+	return s3.New(sess)
+
+}