From 1435282cb0beb923696a99b05e77384b2c2ecfdf Mon Sep 17 00:00:00 2001 From: benduncan <5178243+benduncan@users.noreply.github.co> Date: Sat, 8 Nov 2025 23:32:58 +1100 Subject: [PATCH] Improved AMI import feature, WIP --- INSTALL.md | 28 +++++++++++++++++- cmd/hive/cmd/admin.go | 69 ++++++++++++++++++++++++++++++++++++++++++- go.mod | 2 +- go.sum | 2 ++ hive/utils/utils.go | 26 ++++++++++++++++ 5 files changed, 124 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 6e8caea5e..19951b5b7 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -138,7 +138,33 @@ Validate the new key is available } ``` -## Create AMI template +## Create AMI template (new method) + +Use the hive CLI tool to import a selected OS image. Note, the source file can be compressed (e.g image.tar.gz, image.gz, image.tar.xz) and the tool will automatically extract and upload the raw OS image as an AMI. + +Note, when downloading OS images, use supported platforms that support the `cloud-init` feature to automatically bootstrap when using the Hive EC2 functionality. + +Example, download the Debian 12 image from the repository [https://cloud.debian.org/images/cloud/bookworm/latest/](https://cloud.debian.org/images/cloud/bookworm/latest/) + +Download the image: + +``` +wget https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.tar.xz -O ~/debian-12-genericcloud-arm64.tar.xz +``` + +Import as an AMI to the backend store: + +``` +./bin/hive admin images import --file ~/debian-12-genericcloud-arm64.tar.xz --arch aarch64 --distro debian --version 12 +``` + +Next, verify available disk images to confirm the import was successful + +``` +aws ec2 describe-images +``` + +## Create AMI template (old method) ### Step 1 diff --git a/cmd/hive/cmd/admin.go b/cmd/hive/cmd/admin.go index d18de249d..5b722aab5 100644 --- a/cmd/hive/cmd/admin.go +++ b/cmd/hive/cmd/admin.go @@ -32,7 +32,11 @@ import ( "text/template" "time" + "github.com/mulgadc/hive/hive/config" + "github.com/mulgadc/hive/hive/utils" "github.com/mulgadc/viperblock/viperblock" + "github.com/mulgadc/viperblock/viperblock/backends/s3" + vutils "github.com/mulgadc/viperblock/viperblock/utils" "github.com/spf13/cobra" "gopkg.in/ini.v1" ) @@ -117,6 +121,15 @@ func init() { func runimagesImportCmd(cmd *cobra.Command, args []string) { + cfgFile, _ := cmd.Flags().GetString("config") + + if cfgFile == "" { + + homeDir, _ := os.UserHomeDir() + cfgFile = fmt.Sprintf("%s/hive/config/hive.toml", homeDir) + + } + imageFile, _ := cmd.Flags().GetString("file") if imageFile == "" { @@ -183,7 +196,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { manifest := viperblock.VolumeConfig{} // Calculate the size - manifest.AMIMetadata.Name = fmt.Sprintf("%s-%s-%s-ami", distro, version, arch) + manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", distro, version, arch) manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Hive", manifest.AMIMetadata.Name) manifest.AMIMetadata.Architecture = arch @@ -222,6 +235,60 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { // Next, validate if the image is raw, tar, gz, xv, etc. We need to upload the raw image + tmpDir, err := os.MkdirTemp("", "hive-image-tmp-*") + + if err != nil { + fmt.Fprintf(os.Stderr, "Could not create temp dir: %v\n", err) + os.Exit(1) + } + + imagePath, err = utils.ExtractDiskImageFromFile(imageFile, tmpDir) + + if err != nil { + fmt.Fprintf(os.Stderr, "Could not extract image: %v\n", err) + os.Exit(1) + } + + // Upload the image to S3 (predastore) + + appConfig, err := config.LoadConfig(cfgFile) + + if err != nil { + fmt.Println("Error loading config file:", err) + return + } + + s3Config := s3.S3Config{ + VolumeName: manifest.AMIMetadata.Name, + VolumeSize: uint64(imageStat.Size()), + Bucket: appConfig.Predastore.Bucket, + Region: appConfig.Predastore.Region, + AccessKey: appConfig.AccessKey, + SecretKey: appConfig.SecretKey, + Host: appConfig.Predastore.Host, + } + + vbConfig := viperblock.VB{ + VolumeName: manifest.AMIMetadata.Name, + VolumeSize: uint64(imageStat.Size()), + BaseDir: tmpDir, + Cache: viperblock.Cache{ + Config: viperblock.CacheConfig{ + Size: 0, + }, + }, + VolumeConfig: manifest, + } + + err = vutils.ImportDiskImage(&s3Config, &vbConfig, imagePath) + + if err != nil { + fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err) + os.Exit(1) + } + + defer os.RemoveAll(tmpDir) + } func runAdminInit(cmd *cobra.Command, args []string) { diff --git a/go.mod b/go.mod index a6f8a75de..edc7b2b42 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/google/uuid v1.6.0 github.com/kdomanski/iso9660 v0.4.0 github.com/mulgadc/predastore v0.1.5 - github.com/mulgadc/viperblock v0.1.0 + github.com/mulgadc/viperblock v0.1.1 github.com/nats-io/nats-server/v2 v2.11.6 github.com/nats-io/nats.go v1.43.0 github.com/spf13/cobra v1.9.1 diff --git a/go.sum b/go.sum index 7f437b04e..aa7368b0c 100644 --- a/go.sum +++ b/go.sum @@ -51,6 +51,8 @@ github.com/mulgadc/predastore v0.1.5 h1:7SySkQQ5MQ1NTcKcB14pCUIJipq0YVgCJZeKM6yt github.com/mulgadc/predastore v0.1.5/go.mod h1:BhMymRV3zMTu2S+5pJnNDDm5+AGakjhSc/PKX0q8Xfg= github.com/mulgadc/viperblock v0.1.0 h1:vReunbVOrxzlw7n+GI35lDRHdWBR0HeslZun7t7I0Kk= github.com/mulgadc/viperblock v0.1.0/go.mod h1:LPXBqDzIVQFs8VvVkTgAiZ4vQhe079hvB77mYzN6+/g= +github.com/mulgadc/viperblock v0.1.1 h1:R1DVX1S+/xPPMQ4LIFkwIX7Es+RS3Wgdd8bsWLTEvgY= +github.com/mulgadc/viperblock v0.1.1/go.mod h1:zCjdQ76fi2Qs2a5ign9A/Uqgbz77xts5WV0OR1be48I= github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= github.com/nats-io/nats-server/v2 v2.11.6 h1:4VXRjbTUFKEB+7UoaKL3F5Y83xC7MxPoIONOnGgpkHw= diff --git a/hive/utils/utils.go b/hive/utils/utils.go index 7c5ef74da..09a6f34ea 100644 --- a/hive/utils/utils.go +++ b/hive/utils/utils.go @@ -3,11 +3,13 @@ package utils import ( "bufio" "bytes" + "crypto/tls" "encoding/json" "encoding/xml" "errors" "fmt" "log/slog" + "net/http" "os" "os/exec" "path/filepath" @@ -18,8 +20,12 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/mulgadc/hive/hive/config" ) func ReadPidFile(name string) (int, error) { @@ -486,3 +492,23 @@ func validateDiskImagePath(diskimage string) (err error) { return errors.New("no valid disk image found") } + +func CreateS3Client(cfg *config.Config) *s3.S3 { + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + httpClient := &http.Client{Transport: tr} + + sess := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(cfg.Predastore.Region), + Endpoint: aws.String(fmt.Sprintf("https://%s", cfg.Predastore.Host)), + Credentials: credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, ""), + S3ForcePathStyle: aws.Bool(true), + DisableSSL: aws.Bool(false), + HTTPClient: httpClient, + })) + + return s3.New(sess) + +}