WIP, moving to new gateway model for service control

This commit is contained in:
benduncan
2025-10-07 20:27:31 +08:00
parent cc8ee46592
commit cc28dd0768
11 changed files with 440 additions and 144 deletions
-1
View File
@@ -13,7 +13,6 @@ jobs:
- name: Checkout Code
uses: actions/checkout@v4
# this is needed as easyjson requires GOPATH to be set
- name: Setup Go
uses: actions/setup-go@v5
with:
+6 -50
View File
@@ -4,12 +4,12 @@ import (
"bytes"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"log/slog"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/hive/hive/utils"
)
/*
@@ -22,21 +22,6 @@ b'<?xml version="1.0" encoding="UTF-8"?>\n<Response><Errors><Error><Code>Invalid
*/
type ErrorResponse struct {
XMLName xml.Name `xml:"Response"`
Errors Errors `xml:"Errors"`
RequestID string `xml:"RequestID"`
}
type Errors struct {
Error ErrorDetail `xml:"Error"`
}
type ErrorDetail struct {
Code string `xml:"Code"`
Message string `xml:"Message"`
}
type runInstancesResponse struct {
XMLName xml.Name `xml:"RunInstancesResponse"`
Xmlns string `xml:"xmlns,attr"`
@@ -52,31 +37,6 @@ type responseMetadata struct {
RequestID string `xml:"RequestId"`
}
func GenerateEC2ErrorResponse(code, message, requestID string) (output []byte) {
errorXml := ErrorResponse{
Errors: Errors{
Error: ErrorDetail{
Code: code,
Message: message,
},
},
RequestID: requestID,
}
output, err := xml.MarshalIndent(errorXml, "", " ")
if err != nil {
slog.Error("Failed to build XML", "error", err)
return nil
}
// Add XML header
output = append([]byte(xml.Header), output...)
return output
}
func EC2_Process_RunInstances(jsonData []byte) (output []byte, err error) {
var input ec2.RunInstancesInput
@@ -159,15 +119,11 @@ func EC2_RunInstances(input *ec2.RunInstancesInput) (output bytes.Buffer, err er
},
}
var buf bytes.Buffer
enc := xml.NewEncoder(&buf)
enc.Indent("", " ")
if err := xmlutil.BuildXML(payload, enc); err != nil {
panic(err)
output, err = utils.MarshalToXML(payload)
if err != nil {
return output, errors.New("failed to marshal response to XML")
}
enc.Flush()
return buf, nil
return output, nil
}
+2
View File
@@ -53,6 +53,7 @@ func TestInvalidRunInstance(t *testing.T) {
spew.Dump(response)
}
/*
func TestGenerateEC2ErrorResponse(t *testing.T) {
errorCode := "InvalidInstanceID.NotFound"
@@ -63,3 +64,4 @@ func TestGenerateEC2ErrorResponse(t *testing.T) {
assert.Contains(t, string(xmlResponse), "InvalidInstanceID.NotFound")
}
*/
+22 -23
View File
@@ -1,47 +1,44 @@
package gateway
import (
"fmt"
"log/slog"
"errors"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/davecgh/go-spew/spew"
"github.com/gofiber/fiber/v2"
"github.com/mulgadc/hive/hive/awsec2query"
"github.com/mulgadc/hive/hive/awsparser"
gateway_ec2_core "github.com/mulgadc/hive/hive/gateway/ec2/core"
gateway_ec2_instance "github.com/mulgadc/hive/hive/gateway/ec2/instance"
)
func (gw *GatewayConfig) EC2_Request(ctx *fiber.Ctx) error {
fmt.Println("HERE!")
queryArgs := ParseAWSQueryArgs(string(ctx.Body()))
fmt.Println("Body")
spew.Dump(ctx.Body())
fmt.Println("Request POST")
spew.Dump(ctx.Request())
// Headers
fmt.Println("Headers")
spew.Dump(ctx.GetReqHeaders())
queryArgs := parseAWSQueryArgs(string(ctx.Body()))
var xmlOutput []byte
var err error
// Run the action
// TODO: Generate for each action, unit test each, and invalid action
switch queryArgs["Action"] {
case "DescribeInstances":
return gw.EC2_DescribeInstances(ctx, queryArgs)
xmlOutput, err = gateway_ec2_instance.DescribeInstances(ctx, queryArgs)
case "RunInstances":
return gw.EC2_RunInstances(ctx, queryArgs)
xmlOutput, err = gateway_ec2_instance.RunInstances(ctx, queryArgs)
case "CreateKeyPair":
return gw.EC2_CreateKeyPair(ctx, queryArgs)
xmlOutput, err = gateway_ec2_core.CreateKeyPair(ctx, queryArgs)
default:
slog.Warn("EC2 Unsupported Action", "action", queryArgs["Action"])
err = errors.New("InvalidAction")
}
return fiber.NewError(fiber.StatusNotImplemented, "Action not implemented")
// Return an error XML
if err != nil {
return err
}
ctx.Status(fiber.StatusOK).Type("text/xml").Send(xmlOutput)
return nil
}
/*
func (gw *GatewayConfig) EC2_DescribeInstances(ctx *fiber.Ctx, args map[string]string) error {
slog.Info("EC2 DescribeInstances called")
@@ -186,3 +183,5 @@ func (gw *GatewayConfig) EC2_CreateKeyPair(ctx *fiber.Ctx, args map[string]strin
return nil
}
*/
@@ -0,0 +1,5 @@
package gateway_ec2
func AcceptAddressTransferRequest() {
}
+9
View File
@@ -0,0 +1,9 @@
package gateway_ec2_core
import "github.com/gofiber/fiber/v2"
func CreateKeyPair(ctx *fiber.Ctx, args map[string]string) ([]byte, error) {
return nil, nil
}
@@ -0,0 +1,8 @@
package gateway_ec2_instance
import "github.com/gofiber/fiber/v2"
func DescribeInstances(ctx *fiber.Ctx, args map[string]string) ([]byte, error) {
return nil, nil
}
+228
View File
@@ -0,0 +1,228 @@
package gateway_ec2_instance
import (
"encoding/json"
"errors"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/gofiber/fiber/v2"
"github.com/mulgadc/hive/hive/awsec2query"
"github.com/mulgadc/hive/hive/utils"
)
type runInstancesResponse struct {
Reservation *ec2.Reservation `locationName:"RunInstancesResponse"`
}
type runInstancesResult struct {
Reservation *ec2.Reservation `xml:"reservationSet>item"`
}
/*
Sample JSON:
"RunInstances":{
"name":"RunInstances", // Function name
"http":{
"method":"POST",
"requestUri":"/"
},
"input":{"shape":"RunInstancesRequest"}, // Input shape from AWS API
"output":{"shape":"Reservation"} // Output shape from AWS API
},
"RunInstancesRequest":{
"type":"structure",
"required":[
"MaxCount",
"MinCount"
],
"members":{
"BlockDeviceMappings":{
"shape":"BlockDeviceMappingRequestList",
"locationName":"BlockDeviceMapping"
},
"ImageId":{"shape":"ImageId"},
"InstanceType":{"shape":"InstanceType"},
"Ipv6AddressCount":{"shape":"Integer"},
"Ipv6Addresses":{
"shape":"InstanceIpv6AddressList",
"locationName":"Ipv6Address"
},
"KernelId":{"shape":"KernelId"},
"KeyName":{"shape":"KeyPairName"},
"MaxCount":{"shape":"Integer"},
"MinCount":{"shape":"Integer"},
"Monitoring":{"shape":"RunInstancesMonitoringEnabled"},
"Placement":{"shape":"Placement"},
"RamdiskId":{"shape":"RamdiskId"},
"SecurityGroupIds":{
"shape":"SecurityGroupIdStringList",
"locationName":"SecurityGroupId"
},
"SecurityGroups":{
"shape":"SecurityGroupStringList",
"locationName":"SecurityGroup"
},
"SubnetId":{"shape":"SubnetId"},
"UserData":{"shape":"RunInstancesUserData"},
"AdditionalInfo":{
"shape":"String",
"locationName":"additionalInfo"
},
"ClientToken":{
"shape":"String",
"idempotencyToken":true,
"locationName":"clientToken"
},
"DisableApiTermination":{
"shape":"Boolean",
"locationName":"disableApiTermination"
},
"DryRun":{
"shape":"Boolean",
"locationName":"dryRun"
},
"EbsOptimized":{
"shape":"Boolean",
"locationName":"ebsOptimized"
},
"IamInstanceProfile":{
"shape":"IamInstanceProfileSpecification",
"locationName":"iamInstanceProfile"
},
"InstanceInitiatedShutdownBehavior":{
"shape":"ShutdownBehavior",
"locationName":"instanceInitiatedShutdownBehavior"
},
"NetworkInterfaces":{
"shape":"InstanceNetworkInterfaceSpecificationList",
"locationName":"networkInterface"
},
"PrivateIpAddress":{
"shape":"String",
"locationName":"privateIpAddress"
},
"ElasticGpuSpecification":{"shape":"ElasticGpuSpecifications"},
"ElasticInferenceAccelerators":{
"shape":"ElasticInferenceAccelerators",
"locationName":"ElasticInferenceAccelerator"
},
"TagSpecifications":{
"shape":"TagSpecificationList",
"locationName":"TagSpecification"
},
"LaunchTemplate":{"shape":"LaunchTemplateSpecification"},
"InstanceMarketOptions":{"shape":"InstanceMarketOptionsRequest"},
"CreditSpecification":{"shape":"CreditSpecificationRequest"},
"CpuOptions":{"shape":"CpuOptionsRequest"},
"CapacityReservationSpecification":{"shape":"CapacityReservationSpecification"},
"HibernationOptions":{"shape":"HibernationOptionsRequest"},
"LicenseSpecifications":{
"shape":"LicenseSpecificationListRequest",
"locationName":"LicenseSpecification"
},
"MetadataOptions":{"shape":"InstanceMetadataOptionsRequest"},
"EnclaveOptions":{"shape":"EnclaveOptionsRequest"},
"PrivateDnsNameOptions":{"shape":"PrivateDnsNameOptionsRequest"},
"MaintenanceOptions":{"shape":"InstanceMaintenanceOptionsRequest"},
"DisableApiStop":{"shape":"Boolean"},
"EnablePrimaryIpv6":{"shape":"Boolean"}
}
},
*/
// AUTO-GENERATED: RunInstances
// Generated from Function name: RunInstances
func RunInstances(ctx *fiber.Ctx, args map[string]string) ([]byte, error) {
// Check required args from JSON above
// required:[
// "MaxCount",
// "MinCount"
// ]
if _, ok := args["MaxCount"]; !ok {
return nil, errors.New("InvalidParameter")
}
if _, ok := args["MinCount"]; !ok {
return nil, errors.New("InvalidParameter")
}
// Generated from input shape: RunInstancesRequest
var input = &ec2.RunInstancesInput{}
awsec2query.QueryParamsToStruct(args, input)
// Marshal to JSON, to send over the wire for processing (NATS)
jsonData, err := json.Marshal(input)
if err != nil {
return nil, fmt.Errorf("failed to marshal input to JSON: %w", err)
}
// Run the simulated JSON request via NATS, which will return a JSON response
jsonResp, err := EC2_Process_RunInstances(jsonData)
if err != nil {
return nil, fmt.Errorf("failed to process RunInstances request: %w", err)
}
// Unmarshal the JSON response back into a Reservation struct
var reservation ec2.Reservation
err = json.Unmarshal(jsonResp, &reservation)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal JSON response to Reservation: %w", err)
}
// Convert to XML
payload := runInstancesResponse{
Reservation: &reservation,
}
output, err := utils.MarshalToXML(payload)
if err != nil {
return output.Bytes(), errors.New("failed to marshal response to XML")
}
return output.Bytes(), nil
}
func EC2_Process_RunInstances(jsonData []byte) (output []byte, err error) {
var input ec2.RunInstancesInput
err = json.Unmarshal(jsonData, &input)
if err != nil {
return output, fmt.Errorf("failed to unmarshal JSON to RunInstancesInput: %w", err)
}
// Here you would add the logic to actually create the instance in your system.
// For this example, we'll just create a dummy response.
instance := &ec2.Instance{
InstanceId: aws.String("i-0123456789abcdef0"),
State: &ec2.InstanceState{
Code: aws.Int64(16),
Name: aws.String("running"),
},
ImageId: input.ImageId,
InstanceType: input.InstanceType,
KeyName: input.KeyName,
SubnetId: input.SubnetId,
}
reservation := &ec2.Reservation{
Instances: []*ec2.Instance{instance},
OwnerId: aws.String("123456789012"),
}
// Return as JSON, to simulate the NATS response
jsonResponse, err := json.Marshal(reservation)
if err != nil {
return output, fmt.Errorf("failed to marshal reservation to JSON: %w", err)
}
return jsonResponse, nil
}
+42
View File
@@ -0,0 +1,42 @@
package gateway
type ErrorMessage struct {
HTTPCode int
Message string
}
// TODO: This list is incomplete and needs to be expanded
var ErrorLookup = map[string]ErrorMessage{
"AuthFailure": {HTTPCode: 403, Message: "The AWS was not able to validate the provided access credentials"},
"AccessDenied": {HTTPCode: 403, Message: "The AWS was not able to validate the provided access credentials"},
"Blocked": {HTTPCode: 403, Message: "The request was blocked. This can happen if your request is suspected to be fraudulent or otherwise illegal."},
"DependencyViolation": {HTTPCode: 400, Message: "The request depends on another action which has not yet completed successfully."},
"DryRunOperation": {HTTPCode: 412, Message: "The request would have succeeded, but the DryRun flag was set."},
"InternalError": {HTTPCode: 500, Message: "The request processing has failed because of an unknown error, exception or failure."},
"InvalidAction": {HTTPCode: 400, Message: "The action or operation requested is invalid. Verify that the action is typed correctly."},
"InvalidClientTokenId": {HTTPCode: 403, Message: "The X.509 certificate or the AWS access key ID provided does not exist in our records."},
"InvalidParameter": {HTTPCode: 400, Message: "The parameter name or value is invalid."},
"InvalidParameterCombination": {
HTTPCode: 400,
Message: "Parameters that must not be used together were used together.",
},
"InvalidParameterValue": {HTTPCode: 400, Message: "The value for a parameter is invalid."},
"InvalidQueryParameter": {HTTPCode: 400, Message: "The query string contains a syntax error or unsupported parameter."},
"MalformedQueryString": {HTTPCode: 400, Message: "The query string contains a syntax error."},
"MissingAction": {HTTPCode: 400, Message: "The request is missing an action or operation parameter."},
"MissingAuthenticationToken": {
HTTPCode: 403,
Message: "The request must contain either a valid (registered) AWS access key ID or X.509 certificate.",
},
"MissingParameter": {HTTPCode: 400, Message: "A required parameter for the specified action is not supplied."},
"OptInRequired": {HTTPCode: 403, Message: "The AWS account is not signed up for the service."},
"RequestExpired": {HTTPCode: 403, Message: "The request has expired."},
"ServiceUnavailable": {HTTPCode: 503, Message: "The request has failed due to a temporary failure of the server."},
"Throttling": {HTTPCode: 429, Message: "The request was denied because it exceeded the request rate limit."},
"UnauthorizedOperation": {
HTTPCode: 403,
Message: "You are not authorized to perform this operation.",
},
"UnsupportedOperation": {HTTPCode: 400, Message: "The action requested is not supported."},
}
+99 -70
View File
@@ -1,6 +1,7 @@
package gateway
import (
"encoding/xml"
"errors"
"log/slog"
"os"
@@ -24,6 +25,21 @@ var supportedServices = map[string]bool{
"account": true,
}
type ErrorResponse struct {
XMLName xml.Name `xml:"Response"`
Errors Errors `xml:"Errors"`
RequestID string `xml:"RequestID"`
}
type Errors struct {
Error ErrorDetail `xml:"Error"`
}
type ErrorDetail struct {
Code string `xml:"Code"`
Message error `xml:"Message"`
}
func (gw *GatewayConfig) SetupRoutes() *fiber.App {
var logLevel slog.Level
@@ -104,6 +120,37 @@ func (gw *GatewayConfig) SetupRoutes() *fiber.App {
func (gw *GatewayConfig) Request(ctx *fiber.Ctx) error {
// Route the request to the appropriate endpoint (e.g EC2, IAM, etc)
svc, err := gw.GetService(ctx)
slog.Info("Request", "service", svc, "method", ctx.Method(), "path", ctx.Path())
if err != nil {
slog.Error("GetService error", "error", err)
return gw.ErrorHandler(ctx, err)
}
switch svc {
case "ec2":
err = gw.EC2_Request(ctx)
case "account":
err = gw.Account_Request(ctx)
case "iam":
err = gw.IAM_Request(ctx)
default:
err = errors.New("UnsupportedOperation")
}
if err != nil {
slog.Error("Service request error", "service", svc, "error", err)
return gw.ErrorHandler(ctx, err)
} else {
slog.Info("Service request completed", "service", svc)
}
return nil
}
func (gw *GatewayConfig) GetService(ctx *fiber.Ctx) (srv string, err error) {
// Determine the service from the Authorization header
// "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20250930/ap-southeast-2/account/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=adb8183545a9d6f5b626908582b4ca5f9a309137b1d4bcc802f0e79ede7af7bc"
@@ -113,107 +160,64 @@ func (gw *GatewayConfig) Request(ctx *fiber.Ctx) error {
parts := strings.Split(authHeader, ", ")
if len(parts) != 3 {
slog.Debug("Invalid Authorization header format")
return errors.New("AccessDenied")
return srv, errors.New("AuthFailure")
}
// Parse credential
creds := strings.Split(strings.TrimPrefix(parts[0], "AWS4-HMAC-SHA256 Credential="), "/")
if len(creds) != 5 {
slog.Debug("Invalid credential scope")
return errors.New("AccessDenied")
return srv, errors.New("AuthFailure")
}
svc := creds[3]
if !supportedServices[svc] {
slog.Debug("Unsupported service", "service", svc)
return errors.New("UnsupportedOperation")
return srv, errors.New("UnsupportedOperation")
}
slog.Info("Request", "service", svc, "method", ctx.Method(), "path", ctx.Path())
switch svc {
case "ec2":
return gw.EC2_Request(ctx)
case "account":
return gw.Account_Request(ctx)
case "iam":
return gw.IAM_Request(ctx)
default:
return errors.New("UnsupportedOperation")
}
return svc, nil
}
func (gw *GatewayConfig) ErrorHandler(ctx *fiber.Ctx, err error) error {
// TODO: Support service type specific errors (e.g EC2, S3, etc)
// TODO: Support service type specific errors (e.g EC2, S3, IAM, differ)
svc, _ := gw.GetService(ctx)
slog.Debug("ErrorHandler", "service", svc, "error", err.Error())
// Status code defaults to 500
httpCode := fiber.StatusInternalServerError
var s3error s3.S3Error
var e *fiber.Error
// Check for specific error types
switch {
case strings.Contains(err.Error(), "NoSuchBucket") || strings.Contains(err.Error(), "Bucket not found"):
// File or bucket not found
httpCode = fiber.StatusNotFound
s3error.Code = "NoSuchBucket"
s3error.Message = "The specified bucket does not exist"
// Get the request ID
var requestId = uuid.NewString()
requestId = ctx.Get("x-amz-request-id", requestId)
case strings.Contains(err.Error(), "AccessDenied") || strings.Contains(err.Error(), "Not enough permissions"):
// Permission error
httpCode = fiber.StatusForbidden
s3error.Code = "AccessDenied"
s3error.Message = "Access Denied"
var errorMsg = ErrorMessage{}
case strings.Contains(err.Error(), "NoSuchObject") || strings.Contains(err.Error(), "not found") ||
errors.Is(err, os.ErrNotExist):
// File not found
httpCode = fiber.StatusNotFound
s3error.Code = "NoSuchKey"
s3error.Message = "The specified key does not exist"
case strings.Contains(err.Error(), "Invalid signature") || strings.Contains(err.Error(), "Invalid access key"):
// Authentication error
httpCode = fiber.StatusForbidden
s3error.Code = "SignatureDoesNotMatch"
s3error.Message = "The request signature does not match"
case strings.Contains(err.Error(), "Missing Authorization header"):
// Missing auth header
httpCode = fiber.StatusForbidden
s3error.Code = "AccessDenied"
s3error.Message = "Access Denied"
case strings.Contains(err.Error(), "UnsupportedOperation"):
// Unsupported operation
httpCode = fiber.StatusBadRequest
s3error.Code = "UnsupportedOperation"
s3error.Message = "The operation is not supported"
ctx.Set("X-Amzn-Errortype", "UnsupportedOperationException")
case errors.As(err, &e):
httpCode = e.Code
s3error.Message = e.Message
s3error.Code = "InternalError"
default:
s3error.Code = "InternalError"
s3error.Message = err.Error()
// Check if the error lookup exists
if _, exists := ErrorLookup[err.Error()]; !exists {
slog.Warn("Unknown error code", "error", err.Error())
err = errors.New("InternalError")
}
// Add request ID and host ID
errorMsg = ErrorLookup[err.Error()]
s3error.RequestId = ctx.GetRespHeader("x-amz-request-id", uuid.NewString())
s3error.HostId = ctx.Hostname()
xmlError := GenerateEC2ErrorResponse(err.Error(), errorMsg.Message, requestId)
slog.Debug("Generated error response", "error", err.Error(), "xml", string(xmlError), "requestId", requestId)
if errorMsg.HTTPCode == 0 {
errorMsg.HTTPCode = 500
}
// Set standard S3 error response headers
ctx.Set("Content-Type", "application/xml")
return ctx.Status(httpCode).XML(s3error)
return ctx.Status(errorMsg.HTTPCode).Send(xmlError)
}
// Parse AWS query arguments (used by some services like EC2/S3)
func parseAWSQueryArgs(query string) map[string]string {
func ParseAWSQueryArgs(query string) map[string]string {
params := make(map[string]string)
pairs := strings.Split(query, "&")
for _, pair := range pairs {
@@ -226,3 +230,28 @@ func parseAWSQueryArgs(query string) map[string]string {
}
return params
}
func GenerateEC2ErrorResponse(code, message, requestID string) (output []byte) {
errorXml := ErrorResponse{
Errors: Errors{
Error: ErrorDetail{
Code: code,
Message: errors.New(message),
},
},
RequestID: requestID,
}
output, err := xml.MarshalIndent(errorXml, "", " ")
if err != nil {
slog.Error("Failed to build XML", "error", err)
return nil
}
// Add XML header
output = append([]byte(xml.Header), output...)
return output
}
+19
View File
@@ -2,6 +2,7 @@ package utils
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"os"
@@ -9,6 +10,8 @@ import (
"strconv"
"syscall"
"time"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
)
func ReadPidFile(name string) (int, error) {
@@ -198,3 +201,19 @@ func dirExists(path string) bool {
}
return info.IsDir()
}
// Convert interface to XML
func MarshalToXML(payload interface{}) (bytes.Buffer, error) {
var buf bytes.Buffer
enc := xml.NewEncoder(&buf)
enc.Indent("", " ")
if err := xmlutil.BuildXML(payload, enc); err != nil {
panic(err)
}
enc.Flush()
return buf, nil
}