Compare commits

...

253 Commits

Author SHA1 Message Date
github-actions[bot] f050fa35e0 chore: release 1.17.1 2026-07-21 21:29:30 +00:00
Charles GTE 90d55f7630 fix: backup folder name (#86)
* fix: support configurable backup file prefix (#83)

* fix: backup folder name

* fix: ghcr publish agent image

---------

Co-authored-by: Antonin Jousson <18756890+Antoninj@users.noreply.github.com>
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-21 23:27:10 +02:00
github-actions[bot] 82ffb71496 chore: release 1.17.0 2026-07-11 10:08:21 +00:00
Charles GTE 20b3b58024 Merge pull request #80 from Portabase/fix/storage-encryption
fix/storage-encryption
2026-07-11 12:05:59 +02:00
charles-gauthereau 6cddc73565 Merge branch 'main' into fix/storage-encryption
# Conflicts:
#	docker-compose.yml
2026-07-10 18:49:01 +02:00
github-actions[bot] fd3b0076c0 chore: release 1.16.3 2026-07-09 09:11:23 +00:00
Charles GTE 334ab51125 fix: cron (#79)
* fix: cron log

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-09 11:03:55 +02:00
github-actions[bot] 4df3fe2d0c chore: release 1.16.2 2026-07-09 07:38:49 +00:00
Charles GTE ec7715ca26 fix: cron (#78)
* fix: docker-compose.yml

* fix: cron

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-09 09:36:53 +02:00
charles-gauthereau 32d4e22196 fix: status.rs 2026-07-08 22:24:50 +02:00
charles-gauthereau f277c5485f feat: decrypt encrypted storages in status ping 2026-07-08 20:52:17 +02:00
charles-gauthereau 585e0bea72 feat: accept encrypted storages markers in DatabaseStatus 2026-07-08 20:47:20 +02:00
charles-gauthereau ee97c6056a feat: add decrypt_json_gcm for encrypted status storages 2026-07-08 20:43:01 +02:00
charles-gauthereau b37a4f2180 fix: docker-compose.yml 2026-07-07 18:43:30 +02:00
github-actions[bot] 1d15f40662 chore: release 1.16.1 2026-07-04 10:26:50 +00:00
Charles GTE 324a2ea3f2 fix: security (#75) 2026-07-04 12:24:37 +02:00
github-actions[bot] 9ec92af6c7 chore: release 1.16.0 2026-07-03 06:16:42 +00:00
Charles GTE 1dafdca2a0 Merge pull request #74 from Portabase/feat/docker-volume-backup-restore
feat: docker-volume-backup-restore
2026-07-03 08:14:30 +02:00
charles-gauthereau d12817a960 fix: codecov.yml 2026-07-02 22:42:02 +02:00
charles-gauthereau 608b82a254 fix: test docker_volume.rs 2026-07-02 22:08:55 +02:00
charles-gauthereau bbecca92f9 fix: databases.json 2026-07-02 21:51:17 +02:00
charles-gauthereau b2b2733c4d feat: docker-volume provider
refactor: move build_tar to utils::compress

refactor: move choose_restore_path to utils::common

docs: remove docker-volume README section

fix: keep archive path for multi-file non-docker-volume restores

docs: trim docker-volume README section to essentials

refactor: run docker-volume backup/restore inside spawn_blocking like other providers

docs: enable docker socket, add volume example and security notes

feat: sweep orphaned ephemeral helper containers on startup

feat: docker-volume clean-replace restore via upload_to_container

fix: serialize env-var access in docker-volume tests to avoid setenv/getenv UB

feat: docker-volume backup via download_from_container

feat: docker-volume ping via inspect_volume

fix: return extraction dir for multi-file restore archives

feat: gzip already-tar inputs directly instead of double-wrapping

feat: docker helper (client, self-image, container lifecycle, sweep)

feat: docker-volume provider
2026-07-02 21:45:11 +02:00
github-actions[bot] 2c257c5a58 chore: release 1.15.0 2026-06-29 17:14:03 +00:00
Charles GTE c24c0d7058 Merge pull request #73 from Portabase/fix/postgres-ownership
fix: postgres-ownership
2026-06-29 19:12:01 +02:00
charles-gauthereau 34f353b68f fix: postgres keep_ownership flag 2026-06-29 19:10:49 +02:00
charles-gauthereau ee51996401 chore: document keep_ownership option in example databases.json 2026-06-29 18:54:01 +02:00
charles-gauthereau 694b463787 feat: conditionally omit --no-owner/--no-privileges in pg_restore based on keep_ownership option 2026-06-29 18:52:37 +02:00
charles-gauthereau aed1e86956 feat: add generic options map to DatabaseConfig
Add `options: HashMap<String, serde_json::Value>` to DatabaseConfig and
`options: Option<HashMap<...>>` to InputDatabaseConfig so callers can
pass arbitrary per-database options (e.g. keep_ownership for pg_dumpall).
Update all DatabaseConfig struct literals in test files and add three
new config-service unit tests covering present, absent, and non-bool values.
2026-06-29 18:49:01 +02:00
charles-gauthereau 6388aff1e3 fix: postgres cluster dump command 2026-06-29 17:10:11 +02:00
github-actions[bot] 39a77b18a4 chore: release 1.14.1 2026-06-29 13:22:10 +00:00
Charles GTE e62167182e Merge pull request #72 from Portabase/fix/postgres-cluster
fix: postgres cluster dump command
2026-06-29 15:19:54 +02:00
charles-gauthereau 4be54a614e fix: postgres cluster dump command 2026-06-29 15:18:52 +02:00
github-actions[bot] c5eaa5086c chore: release 1.14.0 2026-06-27 19:16:46 +00:00
Charles GTE 610d443afc Dev (#69)
* fix: pgdump-cluster (#68)

* feat: add as_str/from_str to PostgresDumpFormat

* feat: resolve pg_dumpall/psql binary names

* feat: add include_globals field to database config

* feat: add pg_dumpall/psql globals dump and apply

* feat: add postgres backup bundle (manifest + build + resolve)

* feat: bundle globals into postgres backup when include_globals is set

* feat: replay globals before pg_restore when backup archive is a bundle

* refactor: bind FD restore tempdir guard once to clear unused warnings

* docs: demonstrate include_globals in sample databases.json

* chore: silence test-only re-export warning in non-test builds

* revert: remove include_globals feature, restore plain pg_dump/pg_restore

* feat: add pg_dumpall/psql binary names and is_superuser check

* feat: add postgresql-cluster db type and config parsing

* feat: pg_dumpall cluster backup and psql restore

* feat: route postgresql-cluster through PostgresClusterDatabase

* docs: add postgresql-cluster sample to databases.json

* refactor: split cluster mode into cluster/ module (backup, restore, database)

* test: mirror cluster tests into src/tests/domain/cluster/

* feat: add-gcs-storage-provider (#70)

* fix: refactoring

* chore: add google-cloud-storage and google-cloud-auth deps

* feat: add GCS provider config model

* feat: add GCS credential, client, and stream-source helpers

StreamSource bridges build_stream's Send-only byte stream into the SDK's
StreamingSource (which send_buffered requires to be Send+Sync+'static) via a
bounded mpsc channel, avoiding any change to the shared UploadStream type.

* feat: implement GCS StorageProvider upload

* feat: register google-cloud-storage provider in factory

* test: GCS upload roundtrip against fake-gcs-server

* fix: format GCS bucket as projects/_/buckets/<name> in upload_with_client

write_object rejects the bare bucket id with "malformed bucket name"; the
production provider passed config.bucket_name unformatted, so real uploads
would always fail. Format once in upload_with_client so prod and the
fake-gcs-server test share the corrected path.

* style: cargo fmt GCS provider files

* fix: gcs

* fix: azure (#71)
2026-06-27 21:14:21 +02:00
Charles GTE 16a37e033e fix: azure (#71) 2026-06-27 20:51:47 +02:00
Charles GTE 29ca5a317b feat: add-gcs-storage-provider (#70)
* fix: refactoring

* chore: add google-cloud-storage and google-cloud-auth deps

* feat: add GCS provider config model

* feat: add GCS credential, client, and stream-source helpers

StreamSource bridges build_stream's Send-only byte stream into the SDK's
StreamingSource (which send_buffered requires to be Send+Sync+'static) via a
bounded mpsc channel, avoiding any change to the shared UploadStream type.

* feat: implement GCS StorageProvider upload

* feat: register google-cloud-storage provider in factory

* test: GCS upload roundtrip against fake-gcs-server

* fix: format GCS bucket as projects/_/buckets/<name> in upload_with_client

write_object rejects the bare bucket id with "malformed bucket name"; the
production provider passed config.bucket_name unformatted, so real uploads
would always fail. Format once in upload_with_client so prod and the
fake-gcs-server test share the corrected path.

* style: cargo fmt GCS provider files

* fix: gcs
2026-06-27 20:37:22 +02:00
Charles GTE 01f5d34c96 fix: pgdump-cluster (#68)
* feat: add as_str/from_str to PostgresDumpFormat

* feat: resolve pg_dumpall/psql binary names

* feat: add include_globals field to database config

* feat: add pg_dumpall/psql globals dump and apply

* feat: add postgres backup bundle (manifest + build + resolve)

* feat: bundle globals into postgres backup when include_globals is set

* feat: replay globals before pg_restore when backup archive is a bundle

* refactor: bind FD restore tempdir guard once to clear unused warnings

* docs: demonstrate include_globals in sample databases.json

* chore: silence test-only re-export warning in non-test builds

* revert: remove include_globals feature, restore plain pg_dump/pg_restore

* feat: add pg_dumpall/psql binary names and is_superuser check

* feat: add postgresql-cluster db type and config parsing

* feat: pg_dumpall cluster backup and psql restore

* feat: route postgresql-cluster through PostgresClusterDatabase

* docs: add postgresql-cluster sample to databases.json

* refactor: split cluster mode into cluster/ module (backup, restore, database)

* test: mirror cluster tests into src/tests/domain/cluster/
2026-06-27 17:23:57 +02:00
github-actions[bot] 0b7b7ef59b chore: release 1.13.1 2026-06-25 12:58:08 +00:00
Charles GTE 038c5523b7 fix: download-crash-large-files (#67)
* fix(restore): add ProgressTracker for download progress logging

* fix(restore): stream backup download to disk instead of buffering in RAM

* chore(restore): add download diagnostics + exact byte logging

* refactor(restore): drop ProgressTracker, keep streaming download

* feat(restore): drive download progress from RestoreInfo.size

* fix(restore): accept integer or string for RestoreInfo.size

* fix(restore): show raw bytes in download progress log

* fix(restore): collapse progress to highest milestone per chunk

* fix: downloader.rs
2026-06-25 14:55:49 +02:00
github-actions[bot] ea010ce713 chore: release 1.13.0 2026-06-21 14:41:22 +00:00
Charles GTE 8897568281 Merge pull request #66 from Portabase/feat/az-blob-storage-2
feat: az-blob-storage
2026-06-21 16:39:12 +02:00
Charles GTE 66cad4e12e fix: az blog storage tests 2026-06-21 16:22:51 +02:00
Charles GTE 221ed4e7e1 feat: add azure blob storage support 2026-06-20 19:38:00 +02:00
Charles GTE 84735ac399 feat(storage): spike Azure Blob SAS auth + block round-trip against Azurite
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-20 17:49:54 +02:00
Charles GTE f7639de096 build: add azure_storage_blob and azure_core dependencies 2026-06-20 17:39:10 +02:00
github-actions[bot] e504d09cb3 chore: release 1.12.3 2026-06-20 13:00:11 +00:00
Charles GTE 0363b300a4 Merge pull request #65 from Portabase/dev
Dev
2026-06-20 14:58:03 +02:00
Charles GTE bf6e7d41ab fix: Dockerfile 2026-06-20 14:40:37 +02:00
Charles GTE 2c7065ae95 Merge pull request #64 from pcrosthwaite/feature-backup-timeout-on-large-db
Support large MySQL and MariaDb databases
2026-06-20 14:38:26 +02:00
Charles GTE 1fece577cc fix: tests and config.rs and backup.rs args 2026-06-20 14:36:09 +02:00
Charles GTE 273475fa3b Merge branch 'dev' into feature-backup-timeout-on-large-db 2026-06-20 13:02:14 +02:00
Charles GTE 628c017584 Merge pull request #63 from tobfel/tf-main
docs: add Windows build and run guide / and fix(postgres): resolve pg_dump/pg_restore path cross-platform
2026-06-20 12:53:31 +02:00
Charles GTE f02218708a fix: windows-release.yml 2026-06-20 12:52:53 +02:00
Charles GTE 16152328b0 fix: extract the env variable to the settings.rs and some refactoring, and the trigger in windows-release.yml 2026-06-20 12:31:01 +02:00
Peter Crosthwaite 7328827435 Fix mysqldump timeout on large database backups
- Add mysql:8.4 client tools stage to Dockerfile so the real mysqldump
  binary (with --network-timeout) takes PATH precedence over MariaDB's
  mysqldump symlink, which lacks this flag
- Use --network-timeout in MySQL backup command; sets net_read_timeout and
  net_write_timeout to 86400s for the duration of the dump
- Simplify config: replace net_read_timeout/net_write_timeout/max_allowed_packet
  with a single max_packet_size field (optional, defaults to 512M)
- Log mysqldump/mariadb-dump client version as a debug message before each backup
2026-06-20 18:06:21 +10:00
Peter Crosthwaite 348eaac81b Added config options for setting timeouts on mysql/mariadb 2026-06-20 07:51:18 +10:00
tf 0f6c93ecd0 fix encoding 2026-06-19 16:31:21 +02:00
tf befec0deae Windows Build workflow 2026-06-19 16:01:45 +02:00
tf 2052ab0ff5 fix(postgres): resolve pg_dump/pg_restore path cross-platform 2026-06-19 15:27:45 +02:00
tf 461e92d67e docs: add Windows build and run guide 2026-06-19 15:22:24 +02:00
github-actions[bot] 23678bf2d6 chore: release 1.12.2 2026-06-19 10:00:23 +00:00
Charles GTE 02105a8171 Merge pull request #62 from Portabase/fix/password-slash
fix: password-slash
2026-06-19 11:58:26 +02:00
charles-gauthereau f94656a39b fix: postgres password slash 2026-06-19 11:29:51 +02:00
Charles GTE b94ff4e987 Merge branch 'main' into dev 2026-06-09 19:18:38 +02:00
github-actions[bot] dc32c442f3 chore: release 1.12.1 2026-06-07 18:06:37 +00:00
Charles GTE 1453851555 Merge pull request #61 from Portabase/fix/cpu-ram-crash
fix: cpu-ram-crash
2026-06-07 20:04:25 +02:00
github-actions[bot] 185a7ee612 chore: release 1.12.0 2026-06-07 15:46:22 +00:00
Charles GTE d8328f60ca Merge pull request #60 from Portabase/feat/backup-restore-job-logs
feat: backup-restore-job-logs
2026-06-07 17:44:26 +02:00
charles-gauthereau 82c7122790 fix: refactoring 2026-06-07 17:32:16 +02:00
Charles GTE fba72f5c26 feat: add job logs for restoration process 2026-06-07 11:16:02 +02:00
Charles GTE b537980c71 fix: test ci 2026-06-06 21:14:38 +02:00
Charles GTE 0261a07fc2 fix: test ci 2026-06-06 20:59:30 +02:00
Charles GTE c840f91680 fix: test ci 2026-06-06 20:20:52 +02:00
Charles GTE 2d52eccbb2 fix: test ci 2026-06-06 19:43:54 +02:00
Charles GTE f9bf481c79 fix: test ci 2026-06-06 19:26:44 +02:00
Charles GTE 3c853ee7ca fix: tests and refactoring 2026-06-06 19:15:22 +02:00
Charles GTE 45d791358c fix: tracing logs 2026-06-06 11:01:55 +02:00
Charles GTE 0b915d87c9 feat: job logs 2026-06-06 10:22:27 +02:00
Charles GTE 531a25f292 Merge branch 'main' into dev 2026-05-24 16:52:40 +02:00
Charles GTE 41e2b197cb Merge remote-tracking branch 'origin/main' 2026-05-24 16:52:35 +02:00
github-actions[bot] f149363b12 chore: release 1.11.1 2026-05-17 16:51:27 +00:00
Charles GTE aa3ff39bb3 fix: Dockerfile (#58)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

* fix: tests and refactoring, add justfile instead of makefile

* fix: databases.json

* fix: add gbak in production Dockerfile

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-05-17 18:49:18 +02:00
Charles GTE 740ee43038 fix: add gbak in production Dockerfile 2026-05-17 18:26:29 +02:00
Charles GTE 1fccc0bc19 fix: databases.json 2026-05-15 22:17:02 +02:00
Charles GTE 2c15ea64ae Merge branch 'main' into dev 2026-05-15 22:09:41 +02:00
Charles GTE 18bfdf3b7e Merge remote-tracking branch 'origin/main' 2026-05-11 22:09:18 +02:00
github-actions[bot] db26910679 chore: release 1.11.0 2026-05-11 20:07:01 +00:00
Charles GTE eac2853b83 feat: add-mssql-support (#56)
* fix: tests and refactoring, add justfile instead of makefile

* chore: add tiberius dependency for MSSQL support

* feat: add Mssql variant to DbType enum and config validation

* feat: add MSSQL domain adapter with tiberius ping and sqlpackage backup/restore

* feat: install sqlpackage in Docker base and prod stages

* feat: add MSSQL service to docker-compose and config examples

* fix: log backup errors instead of silently discarding them

* fix: use dotnet-install.sh instead of Microsoft apt feed (SHA1 key rejected by Debian Trixie)

* fix: add /TrustServerCertificate:true to sqlpackage commands for self-signed certs

* fix: use connection string with TrustServerCertificate for sqlpackage

* fix: use user database in MSSQL config example (system databases cannot be exported)

* feat: add seed-mssql task and seed script

* fix: docker-compose.yml
2026-05-11 22:04:36 +02:00
Charles GTE 8690a15888 Merge remote-tracking branch 'origin/main' 2026-05-11 19:45:48 +02:00
github-actions[bot] a4bbb37a4e chore: release 1.10.1 2026-04-29 20:42:31 +00:00
Charles GTE ac8f7fd8d8 Merge branch 'main' into dev 2026-04-29 22:41:16 +02:00
Charles GTE c6baa8fdc3 Merge remote-tracking branch 'origin/main'
# Conflicts:
#	docker-compose.yml
2026-04-29 22:40:31 +02:00
Charles GTE 14068846b9 fix: migration-process (#55)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

* fix: tests and refactoring, add justfile instead of makefile

* fix: restoration process for postgres/mysql and mariadb

* fix: restoration process for postgres/mysql and mariadb

* fix: postgres restore

* fix: mariadb/mysql restore, added backticks to drop command

* fix: mongodb and migration test for firebird

* fix: docker-compose.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-29 22:40:02 +02:00
Charles GTE 6edf2890f1 Merge remote-tracking branch 'origin/dev' into dev 2026-04-26 17:42:52 +02:00
Charles GTE 5f579690ff Merge branch 'main' into dev 2026-04-26 17:42:46 +02:00
Charles GTE bfe63d7fd3 fix: tests and refactoring, add justfile instead of makefile 2026-04-26 17:42:30 +02:00
github-actions[bot] 0a422448b4 chore: release 1.10.0 2026-04-16 09:22:34 +00:00
Charles GTE 14a6c9ab0a feat: add chunk size in .env variables (#54)
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-16 11:20:14 +02:00
charles-gauthereau 5c35375df7 Merge branch 'main' into dev 2026-04-08 19:57:08 +02:00
Charles GTE 86027fd6bd fix: docker compose files [skip-release] (#53)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-02 19:30:07 +02:00
charles-gauthereau 61c7224104 chore: docker-compose.databases.yml 2026-04-02 19:28:20 +02:00
charles-gauthereau d7958e03b9 Merge branch 'main' into dev 2026-04-01 17:25:32 +02:00
Charles GTE 8eab97a3d6 chore: README.md (#52) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-01 17:24:48 +02:00
github-actions[bot] 25c95108f0 chore: release 1.9.0 2026-03-31 13:12:45 +00:00
Charles GTE f37e4d2de9 feat: add firebird (#51)
* feat: add firebird

* feat: add firebird tests

* fix: tests and refactoring, add justfile instead of makefile
2026-03-31 15:10:05 +02:00
Charles GTE b269d98d3c Merge branch 'main' into dev 2026-03-29 09:32:25 +02:00
github-actions[bot] a1aa66ff3f chore: release 1.8.0 2026-03-29 07:30:57 +00:00
Charles GTE d4abcbcebc feat: separate mysql and mariadb (#49)
* feat: separate mysql and mariadb

* feat: separate mysql and mariadb
2026-03-29 09:28:30 +02:00
Charles GTE 2d44b62844 Merge branch 'main' into dev 2026-03-28 17:23:31 +01:00
github-actions[bot] 8bf6503f1f chore: release 1.7.0 2026-03-28 15:47:43 +00:00
Charles GTE fdf35228d4 feat: add-heathcheck-database (#48)
feat: add-heathcheck-database
2026-03-28 16:45:13 +01:00
Charles GTE 1855c5120a fix: docker-compose.yml 2026-03-28 16:44:07 +01:00
Charles GTE 3d5a7eeded feat: add health check ping for databases 2026-03-28 16:07:37 +01:00
Charles GTE 517f3779cd feat: sending database ping result for each databases to portabase server 2026-03-27 23:07:06 +01:00
github-actions[bot] a1e18754b1 chore: release 1.6.3 2026-03-27 21:27:23 +00:00
Charles GTE 092f760431 Merge branch 'main' into dev 2026-03-27 22:25:48 +01:00
Charles GTE 725312e1d5 fix: tests (#47)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* fix: adding some tests

* fix: adding some tests

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-27 22:24:59 +01:00
Charles GTE 2c5c805308 Merge branch 'main' into dev 2026-03-21 10:20:08 +01:00
github-actions[bot] 8e8ebb5920 chore: release 1.6.2 2026-03-21 07:53:35 +00:00
Charles GTE 8489a6f6e3 fix: adding retry config for s3 provider (#46) 2026-03-21 08:51:15 +01:00
Charles GTE 38a3274c43 Merge branch 'main' into dev 2026-03-21 08:34:41 +01:00
github-actions[bot] 3f5b9c91cc chore: release 1.6.1 2026-03-18 18:42:23 +00:00
Charles GTE 1c52f298d2 fix: Enable Link-Time Optimization (LTO) and codegen-units = 1 for Release builds (#45) 2026-03-18 19:39:49 +01:00
Charles GTE bb45c1961f Merge branch 'main' into dev 2026-03-18 19:11:42 +01:00
github-actions[bot] e602c3540f chore: release 1.6.0 2026-03-16 13:38:02 +00:00
charles-gauthereau ee7016d1fa Merge branch 'main' into dev
# Conflicts:
#	docker-compose.yml
2026-03-16 14:36:05 +01:00
Charles GTE b5e8ccac1f feat: add-valkey #43
feat: add-valkey
2026-03-16 14:35:31 +01:00
charles-gauthereau f14a6dc318 fix: docker-compose.yml 2026-03-16 14:32:33 +01:00
charles-gauthereau e1bc77e481 fix: valkey.rs 2026-03-16 14:31:11 +01:00
charles-gauthereau cbc7517603 fix: README.md with Valkey content 2026-03-16 14:28:24 +01:00
charles-gauthereau fc4e4e3881 fix: log in backup.rs 2026-03-16 12:32:57 +01:00
charles-gauthereau 5ec7bc9f28 fix: docker-compose.yml volume for valkey 2026-03-16 12:32:12 +01:00
charles-gauthereau 45c33dd031 feat: add valkey test sequences 2026-03-16 12:30:22 +01:00
charles-gauthereau cddb9d78fa fix: entrypoint.sh 2026-03-16 12:16:03 +01:00
charles-gauthereau cf5de3115c feat: add valkey database engine for backup 2026-03-16 12:11:01 +01:00
charles-gauthereau d6d2ef18f8 fix: internal redis port 2026-03-16 11:24:30 +01:00
github-actions[bot] 987c630c73 chore: release 1.5.1 2026-03-16 07:59:25 +00:00
Charles GTE e9147d6c60 fix: s3 port type mismatch, add string_or_number_to_string deserializer (#42)
Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-16 08:57:00 +01:00
charlesgauthereau c46c301371 fix: s3 port type mismatch, add string_or_number_to_string deserializer 2026-03-16 08:55:16 +01:00
github-actions[bot] 174ead72a3 chore: release 1.5.0 2026-03-15 11:06:29 +00:00
Charles GTE f2a67b3490 feat: add-redis #39
feat: add-redis
2026-03-15 12:04:05 +01:00
charlesgauthereau d4afefbc1e fix: requirements.sh for tests tools 2026-03-14 20:08:26 +01:00
charlesgauthereau be6e147df2 fix: requirements.sh for tests tools 2026-03-14 19:54:15 +01:00
charlesgauthereau 91b6658852 fix: requirements.sh for tests tools 2026-03-14 19:40:01 +01:00
charlesgauthereau f7dcb6b2bb fix: requirements.sh for tests tools 2026-03-14 18:26:26 +01:00
charlesgauthereau ab14f0dcf0 fix: requirements.sh for tests tools 2026-03-14 18:17:43 +01:00
charlesgauthereau 58a0b92710 fix: pipeline codecov.yml 2026-03-14 18:05:29 +01:00
charlesgauthereau 2f24291c0e fix: pipeline codecov.yml 2026-03-14 18:02:06 +01:00
charlesgauthereau 1fdea31430 fix: postgres and redis tests 2026-03-14 17:59:44 +01:00
charlesgauthereau d1d633fd00 feat: add redis database support for backup. 2026-03-13 22:44:46 +01:00
Charles GTE b1c14f098b fix: codecov.yml (#38)
* feat: add codecov.yml

* fix: add codecov.yml

* fix: codecov.yml

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-13 16:58:51 +01:00
github-actions[bot] 3abb955e24 chore: release 1.4.1 2026-03-13 15:19:42 +00:00
Charles GTE 6d22be9b50 fix: add codecov.yml (#37) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-13 16:17:09 +01:00
Charles GTE c2df4ba71e fix: readme badge [skip-release] #36
fix: readme badge  [skip-release]
2026-03-13 15:57:16 +01:00
charles-gauthereau 46b3f2466a fix: readme badge 2026-03-13 13:50:30 +01:00
github-actions[bot] a2be03751f chore: release 1.4.0 2026-03-12 18:23:42 +00:00
Charles GTE 7df9df1605 feat : test coverage for utils functions #35
feat : test coverage for utils functions
2026-03-12 19:21:09 +01:00
charlesgauthereau df5b16a153 fix: docker-compose.yml 2026-03-12 19:16:12 +01:00
charlesgauthereau 824a5d52a3 feat: add test for util functions and start working on database providers tests procedures 2026-03-12 18:56:07 +01:00
charlesgauthereau 1496208db4 Merge branch 'main' into dev 2026-03-09 16:46:00 +01:00
charlesgauthereau 454e1b442f Merge remote-tracking branch 'origin/main' 2026-03-09 16:45:52 +01:00
github-actions[bot] 54bca7452a chore: release 1.3.0 2026-03-09 15:43:44 +00:00
Charles GTE a96cef6cb5 feat: helm chart (#34)
* fix: docker.yml

* feat: add helm chart config

* fix: helm config

* feat: add .github workflow for helm chart

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-09 16:41:14 +01:00
charlesgauthereau fd9f183166 Merge branch 'main' into dev 2026-03-08 22:02:36 +01:00
charlesgauthereau 94fb2535ec Merge remote-tracking branch 'origin/main' 2026-03-08 21:59:02 +01:00
github-actions[bot] 189ad866de chore: release 1.2.5 2026-03-08 20:50:53 +00:00
Charles GTE 55bbd4e724 fix: new-workflow-ci (#32)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 21:48:13 +01:00
charlesgauthereau 7fabc9c98c Merge remote-tracking branch 'origin/main' 2026-03-08 21:47:15 +01:00
github-actions[bot] 4f67a57681 chore: release 1.2.4 2026-03-08 20:20:58 +00:00
Charles GTE 45a1118f64 fix: docker build (#31)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 21:18:27 +01:00
charlesgauthereau 063b2e1c2c Merge remote-tracking branch 'origin/main' 2026-03-08 21:12:09 +01:00
github-actions[bot] f2275d5aca chore: release 1.2.3 2026-03-08 20:00:47 +00:00
Charles GTE b136140b55 fix: new-workflow-ci (#30)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:58:19 +01:00
charlesgauthereau 924177a4fe Merge remote-tracking branch 'origin/main' 2026-03-08 20:52:36 +01:00
github-actions[bot] 773077f66b chore: release 1.2.2 2026-03-08 19:48:25 +00:00
Charles GTE 46ff086cf9 fix: new-workflow-ci (#29)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* fix: docker.yml

* fix: docker.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:47:41 +01:00
charlesgauthereau 678df8e2bc fix: docker.yml 2026-03-08 20:46:47 +01:00
github-actions[bot] f563c187db chore: release 1.2.1 2026-03-08 19:43:01 +00:00
Charles GTE df0dffa47b fix: new-workflow-ci (#28)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:42:16 +01:00
github-actions[bot] 864396422b chore: release 1.2.0 2026-03-08 19:38:10 +00:00
Charles GTE e280c1d2f8 fix: workflow (#27)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:31:41 +01:00
Charles GTE 91a9d0654b feat: new workflow CI (#26)
* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:28:26 +01:00
Charles GTE bc857a4c18 feat: new workflow CI
* feat: new workflow CI

* feat: new workflow CI

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:24:26 +01:00
Charles GTE 2454915866 feat: new workflow CI (#24)
Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 20:22:45 +01:00
Charles GTE a4a7a9a2c4 feat: new workflow CI
feat: new workflow CI
2026-03-08 20:12:43 +01:00
charlesgauthereau 997ba2afa2 feat: new workflow CI 2026-03-08 20:12:00 +01:00
charlesgauthereau 7e0c7c0688 Merge branch 'main' into dev
# Conflicts:
#	docker-compose.yml
2026-03-08 19:48:00 +01:00
Charles GTE f973bc1f52 fix: refactoring for backup and restore service
* fix: docker-compose.yml

* fix: refactoring for backup and restore service

* fix: cargo update

* fix: refactoring of the backup and restore services

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-08 19:47:04 +01:00
charlesgauthereau b9b9fcf6ee Merge branch 'main' into dev 2026-03-08 10:02:34 +01:00
charlesgauthereau 0224605c59 fix: docker-compose.yml 2026-03-08 10:02:17 +01:00
Charles GTE 6ea9fdf8bc Merge pull request #21 from Portabase/dev
chore: add coderabbitai config file
2026-03-05 11:21:45 +01:00
Charles GTE ad68be8a5a fix: coderrabbit.yaml 2026-03-05 11:19:47 +01:00
Charles GTE 434c131afb chore: add coderabbitai config file 2026-03-05 11:10:20 +01:00
Charles GTE 675ac5976d Merge pull request #20 from Portabase/dev
chore: add trailing newlines to README.md
2026-03-05 10:59:26 +01:00
Charles GTE 70543fe032 fix: README.md 2026-03-05 10:45:55 +01:00
Charles GTE 77f90308b4 Update README.md 2026-03-02 08:46:32 +01:00
charlesgauthereau 63cb254abd chore: Cargo.lock 2026-02-24 08:45:50 +01:00
charlesgauthereau 7738745def chore(release): 1.1.6 2026-02-24 08:45:26 +01:00
charlesgauthereau 82659f681c feat: add local test db for sqlite 2026-02-23 22:44:57 +01:00
charlesgauthereau eee3bebd50 feat: add local test db for sqlite 2026-02-23 22:43:44 +01:00
charlesgauthereau 58dd0259f9 chore(release): 1.1.6-rc.1 2026-02-23 21:55:47 +01:00
Charles GTE 5b7453208b Merge pull request #16 from Portabase/feat/sqlite
feat/sqlite
2026-02-23 21:54:04 +01:00
charlesgauthereau 35933aae52 fix: refactoring config check on startup agent. 2026-02-23 21:47:18 +01:00
charlesgauthereau 178628cb1c feat: add sqlite support, first working version 2026-02-22 21:46:16 +01:00
charlesgauthereau 3fc60e3f40 fix: s3 provider port config. 2026-02-19 21:16:13 +01:00
charlesgauthereau e62de95827 chore(release): 1.1.5 2026-02-19 21:15:55 +01:00
Charles GTE fc3612fa3f Merge pull request #14 from Portabase/dev
fix: s3 provider port config.
2026-02-19 21:15:19 +01:00
charlesgauthereau 18fe211ad1 fix: s3 provider port config. 2026-02-19 21:14:35 +01:00
charlesgauthereau 74ada01232 chore: Cargo.lock 2026-02-19 16:10:37 +01:00
charlesgauthereau 4fef360c06 chore(release): 1.1.4 2026-02-19 16:10:17 +01:00
charlesgauthereau 2647941008 fix: tus.rs 2026-02-19 16:09:43 +01:00
Charles GTE b4bdc89888 Merge pull request #13 from Portabase/fix/error-handling
fix/error-handling
2026-02-18 19:28:33 +01:00
charlesgauthereau 829e5bade3 fix: tus.rs 2026-02-18 17:28:59 +01:00
charlesgauthereau fc9a4ef8fb chore(release): 1.1.4-rc.1 2026-02-18 17:28:27 +01:00
charlesgauthereau 3c5734ed4e fix: tus.rs 2026-02-18 17:27:24 +01:00
charlesgauthereau 4b77896ecb fix: Cargo.lock 2026-02-16 11:30:38 +01:00
charlesgauthereau 40c4a4e2bb chore(release): 1.1.3 2026-02-16 11:30:16 +01:00
charlesgauthereau 8a05b93ff3 chore(release): 1.1.3-rc.1 2026-02-16 10:56:37 +01:00
charlesgauthereau c186932a79 fix: failed backup 2026-02-16 09:19:31 +01:00
charlesgauthereau bf74173cac chore(release): 1.1.2 2026-02-16 09:19:17 +01:00
charlesgauthereau 3dcebf4fcc fix: failed backup 2026-02-16 09:00:43 +01:00
charlesgauthereau 4de2b81b33 chore(release): 1.1.2-rc.2 2026-02-16 09:00:26 +01:00
charlesgauthereau b06f9d8186 fix: failed backup 2026-02-16 09:00:18 +01:00
charlesgauthereau f5a2182723 fix: blocking for mysql database 2026-02-16 08:43:27 +01:00
charlesgauthereau e8a96980f1 chore(release): 1.1.2-rc.1 2026-02-16 08:43:13 +01:00
charlesgauthereau a64792bcc6 fix: blocking for mysql database 2026-02-16 08:41:44 +01:00
charlesgauthereau 2d4542be5f chore: Cargo.lock 2026-02-16 08:21:32 +01:00
charlesgauthereau 4a00ed574d chore(release): 1.1.1 2026-02-16 08:21:10 +01:00
Charles GTE fd220598fc Merge pull request #12 from Portabase/fix/s3-region
fix/s3-region
2026-02-16 08:19:11 +01:00
charlesgauthereau d8b239a123 fix: s3 region to us-east-1 2026-02-16 08:18:14 +01:00
Charles GTE ebf0526366 Update README.md 2026-02-15 11:52:35 +01:00
Charles GTE 6461c1523b Merge pull request #11 from Portabase/dev
dev
2026-02-15 11:51:51 +01:00
charlesgauthereau 2809373de1 feat: README.md and docker-compose.yml 2026-02-15 11:50:16 +01:00
charlesgauthereau 5349e1fef9 chore(release): 1.1.0 2026-02-13 14:37:18 +01:00
charlesgauthereau 2bcb393d60 feat: Adding AES GCM encryption. 2026-02-13 14:36:54 +01:00
Charles GTE 406273796e Merge pull request #10 from Portabase/dev
dev
2026-02-13 14:36:19 +01:00
charlesgauthereau cb35347cb6 chore(release): 1.1.0-rc.2 2026-02-12 20:24:21 +01:00
Charles GTE b92138fd84 Merge pull request #9 from Portabase/fix/large-backup-v2
fix/large-backup-v2
2026-02-12 20:22:23 +01:00
charlesgauthereau f4c54836f1 feat: Adding AES GCM encryption. 2026-02-12 20:21:07 +01:00
charlesgauthereau e678cf4b06 feat: Adding AES GCM encryption. 2026-02-12 19:49:17 +01:00
charlesgauthereau 3c599e94de feat: Adding AES GCM encryption. 2026-02-12 19:16:52 +01:00
charlesgauthereau 6bbc856f62 feat: Adding AES GCM encryption. 2026-02-12 18:45:38 +01:00
charlesgauthereau 417212c9e8 chore(release): 1.1.0-rc.1 2026-02-10 20:12:05 +01:00
charlesgauthereau 552f1b8813 chore(release): 1.1.0-rc.1 2026-02-10 20:09:08 +01:00
charlesgauthereau e2d7c4747e feat: Ready for Testing. 2026-02-10 20:08:00 +01:00
charlesgauthereau 08b5124a5a feat: added the is_locked check for database. 2026-02-10 19:59:13 +01:00
charlesgauthereau cc0de33a62 fix: working integrating google drive 2026-02-08 20:42:30 +01:00
charlesgauthereau ce00959feb fix: working on restore. 2026-02-08 16:54:45 +01:00
charlesgauthereau cfef9b047b fix: working on metadata update between server and agent. 2026-02-07 23:31:32 +01:00
charlesgauthereau caf6127f7a fix: start working on s3 provider 2026-02-05 22:46:00 +01:00
charlesgauthereau b71f29eb21 fix: fix upload_futures 2026-02-05 08:57:15 +01:00
charlesgauthereau 2988cb297b fix: Working on multiple providers methods 2026-02-04 23:04:31 +01:00
charlesgauthereau 5019e9585b fix: adding the metadata into tasks and also updated them is cron of storages changed 2026-02-04 21:37:53 +01:00
230 changed files with 14740 additions and 1600 deletions
+23
View File
@@ -0,0 +1,23 @@
reviews:
auto_review:
enabled: true
ignore_usernames: ["dependabot[bot]"]
labels: ["!wip", "!draft"]
auto_apply_labels: true
suggested_reviewers: true
auto_assign_reviewers: true
issue_enrichment:
auto_enrich:
enabled: true
labeling:
auto_apply_labels: true
labeling_instructions:
- label: bug
instructions: "Error, crash, or incorrect behavior reports"
- label: enhancement
instructions: "Feature requests or improvements"
- label: documentation
instructions: "Docs updates or missing documentation"
- label: good first issue
instructions: "Beginner-friendly tasks suitable for new contributors"
+1 -1
View File
@@ -60,7 +60,7 @@ representative at an online or offline event.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
contact@soluce-technologies.com.
contact@portabase.io.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
+1 -1
View File
@@ -18,7 +18,7 @@ We take security seriously and aim to support the following versions of the proj
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
1. **Contact Us**
Please report the vulnerability by emailing **[contact@soluce-technologies.com](mailto:contact@soluce-technologies.com)**. Include the following details:
Please report the vulnerability by emailing **[contact@portabase.io](mailto:contact@portabase.io)**. Include the following details:
- A detailed description of the issue.
- Steps to reproduce the vulnerability (if applicable).
- Any potential impacts or risks.
+90
View File
@@ -0,0 +1,90 @@
name: Codecov Rust
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
env:
CARGO_TERM_COLOR: always
CARGO_TARGET_DIR: /app/target
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
jobs:
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build test image
run: docker compose -f docker-compose.test.yml build agent-test
- name: Start agent-test container
run: docker compose -f docker-compose.test.yml up -d agent-test
- name: Run tests inside container
run: |
docker compose -f docker-compose.test.yml exec -T \
-e CARGO_TARGET_DIR \
-e CARGO_INCREMENTAL \
-e RUSTFLAGS \
-e LLVM_PROFILE_FILE \
agent-test bash -c "
mkdir -p /app/coverage &&
rm -rf /app/target/* /app/coverage/* &&
cargo test --verbose -- --test-threads=2 &&
sync
"
- name: Verify profraw files exist
run: |
docker compose -f docker-compose.test.yml exec -T \
agent-test bash -c '
count=$(find /app/coverage -type f -name "*.profraw" | wc -l)
echo "profraw files: $count"
test "$count" -gt 0
'
- name: Generate coverage report inside container
run: |
docker compose -f docker-compose.test.yml exec -T \
agent-test bash -c "
rustup component add llvm-tools-preview &&
grcov /app/coverage \
--binary-path /app/target/debug \
-s /app \
--llvm \
-t lcov \
--branch \
--ignore-not-existing \
--ignore '/app/target/*' \
-o /app/lcov.info
"
- name: Copy lcov.info from container to host
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
- name: Show basic coverage report info
run: |
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
head -n 30 ./lcov.info || true
test -s ./lcov.info
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
files: ./lcov.info
flags: unittests
name: rust-unit-coverage
verbose: true
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
use_pypi: true
- name: Stop and remove container
if: always()
run: docker compose -f docker-compose.test.yml down --volumes
+73
View File
@@ -0,0 +1,73 @@
name: Discord Notification
on:
workflow_call:
inputs:
release_tag:
required: true
type: string
discord_title:
required: true
type: string
discord_color:
required: true
type: number
discord_footer:
required: true
type: string
secrets:
DISCORD_WEBHOOK:
required: true
GH_TOKEN:
required: true
jobs:
notify-discord:
runs-on: ubuntu-latest
steps:
- name: Send Discord Notification
env:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
RELEASE_INFO=$(gh release view "${{ inputs.release_tag }}" -R ${{ github.repository }} --json name,url,body,author)
RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r .name)
if [ -z "$RELEASE_TITLE" ] || [ "$RELEASE_TITLE" = "null" ]; then RELEASE_TITLE="${{ inputs.release_tag }}"; fi
RELEASE_URL=$(echo "$RELEASE_INFO" | jq -r .url)
RELEASE_BODY=$(echo "$RELEASE_INFO" | jq -r .body)
AUTHOR_NAME="Portabase"
AUTHOR_ICON="https://github.com/Portabase.png"
PAYLOAD=$(jq -n \
--arg title "$RELEASE_TITLE" \
--arg description "$RELEASE_BODY" \
--arg url "$RELEASE_URL" \
--arg author "$AUTHOR_NAME" \
--arg icon "$AUTHOR_ICON" \
--arg discord_title "${{ inputs.discord_title }}" \
--arg discord_footer "${{ inputs.discord_footer }}" \
--argjson discord_color ${{ inputs.discord_color }} \
'{
content: $discord_title,
embeds: [{
title: $title,
url: $url,
description: $description,
color: $discord_color,
author: {
name: $author,
icon_url: $icon
},
footer: {
text: $discord_footer
}
}]
}'
)
curl -H "Content-Type: application/json" \
-d "$PAYLOAD" \
"$DISCORD_WEBHOOK"
+67 -45
View File
@@ -3,10 +3,16 @@ name: Docker Publish
on:
workflow_call:
inputs:
version:
required: true
type: string
ref:
required: true
type: string
image_name:
required: false
type: string
default: 'portabase/agent'
default: "portabase/agent"
add_latest:
required: false
type: boolean
@@ -14,11 +20,11 @@ on:
target:
required: false
type: string
default: 'prod'
default: "prod"
dockerfile:
required: false
type: string
default: './docker/Dockerfile'
default: "./docker/Dockerfile"
secrets:
DOCKER_USERNAME:
required: true
@@ -27,32 +33,34 @@ on:
jobs:
build:
name: Build and push Docker images
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' }}
name: Build architectures
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
strategy:
fail-fast: false
matrix:
platform: [linux/amd64, linux/arm64]
platform: [ linux/amd64, linux/arm64 ]
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Set image tag
id: set-tags
- name: Prepare Image Tags
id: prep
run: |
REF_NAME=${GITHUB_REF#refs/tags/}
if [ "${{ matrix.platform }}" = "linux/amd64" ]; then
IMAGE="${{ inputs.image_name }}:$REF_NAME-amd64"
else
IMAGE="${{ inputs.image_name }}:$REF_NAME-arm64"
fi
echo "image=$IMAGE" >> $GITHUB_OUTPUT
ARCH=${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
echo "image=${{ inputs.image_name }}:${{inputs.version}}-$ARCH" >> $GITHUB_OUTPUT
echo "safe_platform=${{ matrix.platform == 'linux/amd64' && 'linux-amd64' || 'linux-arm64' }}" >> $GITHUB_OUTPUT
- name: Build and push image
uses: docker/build-push-action@v6
@@ -61,57 +69,71 @@ jobs:
file: ${{ inputs.dockerfile }}
platforms: ${{ matrix.platform }}
push: true
tags: ${{ steps.set-tags.outputs.image }}
tags: ${{ steps.prep.outputs.image }}
target: ${{ inputs.target }}
- name: Prepare artifact name
id: artifact
run: |
platform=${{ matrix.platform }}
echo "safe_platform=${platform//\//-}" >> $GITHUB_OUTPUT
echo "${{ steps.set-tags.outputs.image }}" > image.txt
- name: Save image name for manifest
run: echo "${{ steps.prep.outputs.image }}" > image.txt
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
- uses: actions/upload-artifact@v4
with:
name: image-${{ steps.artifact.outputs.safe_platform }}
name: image-${{ steps.prep.outputs.safe_platform }}
path: image.txt
if-no-files-found: warn
compression-level: 6
overwrite: false
include-hidden-files: false
retention-days: 1
create-manifest:
name: Create multi-arch Docker manifest
name: Create multi-arch manifest
runs-on: ubuntu-latest
needs: build
steps:
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
- uses: actions/download-artifact@v4
with:
name: image-linux-amd64
path: /tmp/digests/amd64
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
- uses: actions/download-artifact@v4
with:
name: image-linux-arm64
path: /tmp/digests/arm64
- name: Login to Docker
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Generate semantic tags
id: tags
run: |
VERSION="${{ inputs.version }}"
IFS='.' read -r MAJOR MINOR PATCH <<< "$VERSION"
echo "VERSION_TAG=$VERSION" >> $GITHUB_OUTPUT
if [ -n "$MINOR" ]; then
echo "MINOR_TAG=$MAJOR.$MINOR" >> $GITHUB_OUTPUT
fi
if [ -n "$MAJOR" ]; then
echo "MAJOR_TAG=$MAJOR" >> $GITHUB_OUTPUT
fi
if [ "${{ inputs.add_latest }}" = "true" ]; then
echo "LATEST_TAG=latest" >> $GITHUB_OUTPUT
fi
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ inputs.image_name }}
tags: |
type=raw,value=${{ steps.tags.outputs.VERSION_TAG }}
type=raw,value=${{ steps.tags.outputs.MINOR_TAG }}
type=raw,value=${{ steps.tags.outputs.MAJOR_TAG }}
type=raw,value=${{ steps.tags.outputs.LATEST_TAG }}
- name: Create and push manifest list
working-directory: /tmp/digests
run: |
DOCKER_IMAGES="$(cat amd64/image.txt) $(cat arm64/image.txt)"
REF_NAME=${GITHUB_REF#refs/tags/}
MANIFEST_IMAGE="${{ inputs.image_name }}:$REF_NAME"
docker buildx imagetools create $DOCKER_IMAGES -t $MANIFEST_IMAGE
docker buildx imagetools inspect $MANIFEST_IMAGE
if [ "${{ inputs.add_latest }}" = "true" ]; then
docker buildx imagetools create $DOCKER_IMAGES -t ${{ inputs.image_name }}:latest
docker buildx imagetools inspect ${{ inputs.image_name }}:latest
fi
TAG_ARGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")
echo $TAG_ARGS
docker buildx imagetools create $TAG_ARGS $DOCKER_IMAGES
+72
View File
@@ -0,0 +1,72 @@
name: GHCR Publish
on:
workflow_call:
inputs:
version:
required: true
type: string
ref:
required: true
type: string
add_latest:
required: false
type: boolean
default: false
dockerfile:
required: false
type: string
default: "./docker/Dockerfile"
target:
required: false
type: string
default: "prod"
permissions:
contents: read
packages: write
jobs:
publish:
name: Build and push to GHCR
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ghcr.io/${{ github.repository_owner }}/agent
tags: |
type=semver,pattern={{version}},value=${{ inputs.version }}
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
type=semver,pattern={{major}},value=${{ inputs.version }}
type=raw,value=latest,enable=${{ inputs.add_latest }}
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
target: ${{ inputs.target }}
provenance: false
cache-from: type=gha,scope=ghcr-build
cache-to: type=gha,mode=max,scope=ghcr-build,ignore-error=true
-128
View File
@@ -1,128 +0,0 @@
name: GitHub Release
on:
workflow_call:
inputs:
prerelease:
required: false
type: boolean
default: false
make_latest:
required: false
type: boolean
default: false
discord_title:
required: true
type: string
discord_color:
required: true
type: number
discord_footer:
required: true
type: string
secrets:
DISCORD_WEBHOOK:
required: true
GH_TOKEN:
required: true
jobs:
create-release:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Check out the repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Build Changelog
id: build_changelog
uses: mikepenz/release-changelog-builder-action@v5
with:
mode: "COMMIT"
configurationJson: |
{
"template": "#{{CHANGELOG}}",
"categories": [
{
"title": "## Feature",
"labels": ["feat", "feature"]
},
{
"title": "## Fix",
"labels": ["fix", "bug"]
},
{
"title": "## Other",
"labels": []
}
],
"label_extractor": [
{
"pattern": "^(build|chore|ci|docs|feat|fix|perf|refactor|revert|style|test){1}(\([\w\-\.]+\))?(!)?: ([\w ])+([\s\S]*)",
"on_property": "title",
"target": "$1"
}
]
}
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
- name: Create GitHub Release
uses: softprops/action-gh-release@v2
with:
generate_release_notes: false
body: ${{ steps.build_changelog.outputs.changelog }}
prerelease: ${{ inputs.prerelease }}
make_latest: ${{ inputs.make_latest }}
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
- name: Send Discord Notification
env:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
RELEASE_INFO=$(gh release view "${{ github.ref_name }}" -R ${{ github.repository }} --json name,url,body,author)
RELEASE_TITLE=$(echo "$RELEASE_INFO" | jq -r .name)
if [ -z "$RELEASE_TITLE" ] || [ "$RELEASE_TITLE" = "null" ]; then RELEASE_TITLE="${{ github.ref_name }}"; fi
RELEASE_URL=$(echo "$RELEASE_INFO" | jq -r .url)
RELEASE_BODY=$(echo "$RELEASE_INFO" | jq -r .body)
AUTHOR_NAME="Portabase"
AUTHOR_ICON="https://github.com/Portabase.png"
PAYLOAD=$(jq -n \
--arg title "$RELEASE_TITLE" \
--arg description "$RELEASE_BODY" \
--arg url "$RELEASE_URL" \
--arg author "$AUTHOR_NAME" \
--arg icon "$AUTHOR_ICON" \
--arg discord_title "${{ inputs.discord_title }}" \
--arg discord_footer "${{ inputs.discord_footer }}" \
--argjson discord_color ${{ inputs.discord_color }} \
'{
content: $discord_title,
embeds: [{
title: $title,
url: $url,
description: $description,
color: $discord_color,
author: {
name: $author,
icon_url: $icon
},
footer: {
text: $discord_footer
}
}]
}'
)
curl -H "Content-Type: application/json" \
-d "$PAYLOAD" \
"$DISCORD_WEBHOOK"
+37
View File
@@ -0,0 +1,37 @@
name: Publish Helm Chart
on:
workflow_call:
inputs:
version:
required: true
type: string
secrets:
GH_TOKEN:
required: true
jobs:
publish-helm:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v4
- name: Package Helm chart
run: |
mkdir -p ./helm-packages
helm package helm \
--version ${{ inputs.version }} \
--app-version ${{ inputs.version }} \
--destination ./helm-packages
- name: Authenticate to GitHub Packages
run: |
echo "${{ secrets.GH_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push Helm chart to GitHub Packages (OCI)
run: |
helm push ./helm-packages/portabase-agent-${{ inputs.version }}.tgz oci://ghcr.io/portabase/charts
+138 -12
View File
@@ -1,32 +1,158 @@
name: Publish Docker image for release
name: Auto Release & Publish
on:
push:
tags:
- '*.*.*'
- '!*-*'
pull_request:
types: [ closed ]
branches:
- main
permissions:
contents: write
packages: write
jobs:
docker_publish:
check-skip:
runs-on: ubuntu-latest
outputs:
skip: ${{ steps.set-skip.outputs.skip }}
steps:
- name: Determine if release should be skipped
id: set-skip
run: |
TITLE="${{ github.event.pull_request.title }}"
echo "PR title: $TITLE"
if [[ "$TITLE" == *"[skip-release]"* ]]; then
echo "PR title contains [skip-release], skipping release jobs."
echo "skip=true" >> $GITHUB_OUTPUT
else
echo "skip=false" >> $GITHUB_OUTPUT
fi
create-release:
needs: check-skip
if: ${{ needs.check-skip.outputs.skip == 'false' }}
runs-on: ubuntu-latest
outputs:
draft_tag: ${{ steps.release_step.outputs.draft_tag }}
version: ${{ steps.release_step.outputs.version }}
steps:
- uses: actions/create-github-app-token@v1
id: app-token
with:
app-id: ${{ vars.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ steps.app-token.outputs.token }}
ref: main
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
- name: Install cargo-edit
run: cargo install cargo-edit
- uses: actions/setup-node@v4
with:
node-version: "lts/*"
- name: Install release-it globally
run: |
npm install -g release-it
npm install -g @release-it/conventional-changelog
npm install -g @release-it/bumper
- run: |
git config --global user.name 'github-actions[bot]'
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
- name: Run release-it
id: release_step
run: |
git pull origin main
VERSION=$(release-it --ci --release-version)
echo $VERSION
echo "version=$VERSION" >> $GITHUB_OUTPUT
OUTPUT=$(release-it --ci)
echo "$OUTPUT"
DRAFT_TAG=$(echo "$OUTPUT" | grep -oE 'untagged-[a-z0-9]+')
echo $DRAFT_TAG
echo "draft_tag=$DRAFT_TAG" >> $GITHUB_OUTPUT
env:
GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
publish-docker:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/docker.yml
with:
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
add_latest: true
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
github_release:
needs: docker_publish
uses: ./.github/workflows/github.yml
publish-docker-ghcr:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/ghcr.yml
with:
prerelease: false
make_latest: true
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
add_latest: true
publish-helm:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/helm.yml
with:
version: ${{ needs.create-release.outputs.version }}
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
finalize-release:
needs:
- create-release
- publish-docker
- publish-docker-ghcr
- publish-helm
runs-on: ubuntu-latest
outputs:
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Publish GitHub Release
id: publish_release_step
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
OUTPUT=$(gh release edit ${{ needs.create-release.outputs.draft_tag }} --draft=false)
echo "$OUTPUT"
RELEASE_TAG=$(echo "$OUTPUT" | sed -E 's|.*/releases/tag/||')
echo "release_tag=$RELEASE_TAG" >> $GITHUB_OUTPUT
notify-discord:
needs:
- publish-docker
- publish-helm
- create-release
- finalize-release
uses: ./.github/workflows/discord.yml
with:
release_tag: ${{ needs.create-release.outputs.version }}
discord_title: "||@release-agent|| New release published"
discord_color: 5814783
discord_color: 3066993
discord_footer: "Portabase"
secrets:
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
+96
View File
@@ -0,0 +1,96 @@
name: Build Windows release
on:
workflow_dispatch:
push:
tags:
- '[0-9]+.[0-9]+.[0-9]+'
branches:
- main
- master
jobs:
build-windows:
runs-on: windows-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Rust toolchain (MSVC)
uses: actions-rs/toolchain@v1
with:
toolchain: stable-x86_64-pc-windows-msvc
profile: minimal
override: true
- name: Install vcpkg and OpenSSL (x64)
shell: pwsh
run: |
# Install vcpkg and the prebuilt OpenSSL package
git clone https://github.com/microsoft/vcpkg C:\vcpkg
C:\vcpkg\bootstrap-vcpkg.bat
C:\vcpkg\vcpkg install openssl:x64-windows
# Export variables for subsequent steps
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Build (cargo release)
shell: pwsh
env:
# Cargo / openssl-sys will pick up OPENSSL_DIR from the environment
OPENSSL_DIR: ${{ env.OPENSSL_DIR }}
run: |
# Ensure the environment variable is present for this step
if (-Not $env:OPENSSL_DIR) { Write-Host "OPENSSL_DIR not set, printing env for debugging"; Get-ChildItem Env: | ForEach-Object { Write-Host $_ } }
# Build the declared bin target explicitly (Cargo.toml [[bin]] name = "app")
cargo build --release --bin app
- name: Prepare artifact zip
id: prepare_artifact
shell: pwsh
env:
RELEASE_TAG: ${{ github.ref_name }}
run: |
$tag = $env:RELEASE_TAG
if (-not $tag) { $tag = $env:GITHUB_SHA }
# Package the declared bin target deterministically (Cargo.toml [[bin]] name = "app")
$exe = "target\release\app.exe"
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
$outDir = "artifact"
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
# Ship under the package name, not the internal bin name "app"
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
$zipName = "windows-release-$tag.zip"
if (Test-Path $zipName) { Remove-Item $zipName }
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
Write-Host "ZIP=$zipName"
Write-Output "zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
- name: Upload build artifact
uses: actions/upload-artifact@v4
with:
name: windows-release
path: windows-release-*.zip
- name: Create GitHub Release
if: startsWith(github.ref, 'refs/tags/')
id: create_release
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ github.ref_name }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Upload release asset
if: startsWith(github.ref, 'refs/tags/')
uses: actions/upload-release-asset@v1
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: windows-release-${{ github.ref_name }}.zip
asset_name: windows-release-${{ github.ref_name }}.zip
asset_content_type: application/zip
+5 -1
View File
@@ -3,4 +3,8 @@
/src/data/
.DS_Store
.env
.env
.claude
/docs
+62
View File
@@ -0,0 +1,62 @@
{
"github": {
"release": true,
"draft": true,
"tokenRef": "GITHUB_TOKEN"
},
"git": {
"commit": true,
"commitMessage": "chore: release ${version}",
"requireCleanWorkingDir": true,
"tag": true,
"tagName": "${version}",
"push": true
},
"hooks": {
"before:bump": "cargo set-version ${version}"
},
"plugins": {
"@release-it/conventional-changelog": {
"preset": {
"name": "conventionalcommits",
"types": [
{
"type": "feat",
"section": "✨ Features"
},
{
"type": "fix",
"section": "🐛 Bug Fixes"
},
{
"type": "perf",
"section": "⚡️ Performance Improvements"
},
{
"type": "revert",
"section": "⏪️ Reverts"
},
{
"type": "docs",
"section": "📝 Documentation",
"hidden": true
},
{
"type": "chore",
"section": "🔧 Chores",
"hidden": true
}
]
}
},
"@release-it/bumper": {
"out": [
{
"file": "CITATION.cff",
"path": "version",
"type": "text/yaml"
}
]
}
}
}
+9 -4
View File
@@ -1,6 +1,6 @@
cff-version: 1.2.0
title: Portabase Agent (Rust)
message: "If you use this software, please cite it as below."
message: If you use this software, please cite it as below.
type: software
authors:
- family-names: Gauthereau
@@ -9,7 +9,12 @@ authors:
given-names: Killian
repository-code: https://github.com/Portabase/agent-rust
url: https://portabase.io
abstract: "Portabase is a free, open-source, self-hosted solution for database administration, providing backup and restore capabilities, scheduling, retention policies, notifications, and support for multiple storage backends. Its headless agent architecture enables connection to multiple database instances securely and efficiently."
abstract: >-
Portabase is a free, open-source, self-hosted solution for database
administration, providing backup and restore capabilities, scheduling,
retention policies, notifications, and support for multiple storage backends.
Its headless agent architecture enables connection to multiple database
instances securely and efficiently.
keywords:
- database
- administration
@@ -22,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.0.5-rc.2
date-released: "2026-02-01"
version: 1.17.1
date-released: '2026-02-24'
Generated
+3160 -302
View File
File diff suppressed because it is too large Load Diff
+40 -5
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.0.5-rc.2"
version = "1.17.1"
edition = "2024"
[dependencies]
@@ -17,7 +17,7 @@ base64 = "0.22.1"
thiserror = "2.0.17"
log = "0.4.29"
toml = "0.9.10"
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream"] }
reqwest = { version = "0.13.1", features = ["json", "blocking", "multipart", "stream", "query"] }
anyhow = "1.0.100"
tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "fs"] }
async-trait = "0.1.89"
@@ -30,12 +30,47 @@ tokio-postgres = "0.7.15"
futures = "0.3.31"
tracing-appender = "0.2.4"
time = { version = "0.3.44", features = ["macros"] }
mongodb = "3.5.0"
mongodb = "3.5.1"
rand = "0.9.2"
tokio-util = "0.7.18"
bytes = "1.11.0"
async-stream = "0.3.6"
uuid = { version = "1.20.0", features = ["v4"] }
tokio-util = { version = "0.7.18", features = ["compat", "io"] }
tiberius = { version = "0.12", default-features = false, features = ["rustls", "chrono"] }
aws-config = "1.8.13"
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
azure_core = "1.0.0"
azure_storage_blob = "1.0.0"
google-cloud-storage = "1.15"
google-cloud-auth = "1.13"
async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
tokio-tar = "0.3.1"
oauth2 = "5.0.0"
hyper = "1.8.1"
aes-gcm = "0.11.0-rc.3"
generic-array = "0.14.7"
futures-util = "0.3.31"
tokio-stream = "0.1.18"
aes = "0.9.0-rc.4"
typenum = "1.19.0"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
bollard = "0.20.0"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }
mockall = "0.13"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
wiremock = "0.6"
[profile.release]
lto = true
codegen-units = 1
strip = "debuginfo"
[[bin]]
name = "app"
path = "src/main.rs"
path = "src/main.rs"
-33
View File
@@ -1,33 +0,0 @@
include .env
export $(shell sed 's/=.*//' .env)
.PHONY: seed-mongo seed-mysql seed-postgres
seed-mongo:
@echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
@echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed.sql
seed-postgres-1gb:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
+10 -2
View File
@@ -13,14 +13,20 @@
[![License: Apache](https://img.shields.io/badge/License-apache-yellow.svg)](LICENSE)
[![Docker Pulls](https://img.shields.io/docker/pulls/portabase/agent?color=brightgreen)](https://hub.docker.com/r/portabase/agent)
[![Helm Chart](https://img.shields.io/badge/Helm-Kubernetes-326ce5?logo=helm&logoColor=white)](https://github.com/Portabase/agent/pkgs/container/charts%2Fportabase-agent)
[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey)](https://github.com/Portabase/portabase)
[![Support Portabase](https://img.shields.io/badge/Support-Portabase-orange)](https://www.buymeacoffee.com/portabase)
[![PostgreSQL](https://img.shields.io/badge/python-3.13-blue.svg)](https://www.python.org/downloads/release/python-3120/)
[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/)
[![MySQL](https://img.shields.io/badge/MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/)
[![SQLite](https://img.shields.io/badge/-SQLite-blue?logo=sqlite&logoColor=white)](https://sqlite.org/)
[![Redis](https://img.shields.io/badge/Redis-DC382D?style=flat&logo=Redis&logoColor=white)](https://redis.io/)
[![MariaDB](https://img.shields.io/badge/MariaDB-003545?logo=mariadb&logoColor=white)](https://mariadb.org/)
[![MongoDB](https://img.shields.io/badge/-MongoDB-13aa52?logo=mongodb&logoColor=white)](https://www.mongodb.com/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![Firebird](https://img.shields.io/badge/Firebird-f55b14?style=flat&logo=Firebird&logoColor=white)](https://firebirdsql.org/)
[![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase)
[![Open Source](https://img.shields.io/badge/open%20source-❤️-red)](https://github.com/Portabase/portabase)
@@ -48,7 +54,7 @@ You have 4 ways to install Portabase Agent:
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
- Kubernetes with Helm (soon)
- Kubernetes with Helm [details](https://portabase.io/docs/agent/setup#helm)
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
**Ensure Docker is installed on your machine before getting started.**
@@ -79,3 +85,5 @@ Distributed under the Apache License. See `LICENSE.txt` for more information.
[Docker-url]: https://www.docker.com/
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
+97 -5
View File
@@ -10,10 +10,23 @@
"host": "db-postgres",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 1 - PostgreSQL - BIS",
"database": "devdb2",
"type": "postgresql",
"username": "devuser2",
"password": "changeme2",
"port": 5432,
"host": "db-postgres-2",
"generated_id": "16678159-ff7e-5697-8c83-0adeff214681",
"options": {
"keep_ownership": true
}
},
{
"name": "Test database 2 - MariaDB",
"database": "mariadb",
"type": "mysql",
"type": "mariadb",
"username": "mariadb",
"password": "changeme",
"port": 3306,
@@ -21,7 +34,7 @@
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 4 - MongoDB Auth",
"name": "Test database 3 - MongoDB Auth",
"database": "testdbauth",
"type": "mongodb",
"username": "root",
@@ -31,14 +44,93 @@
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 5 - MongoDB",
"name": "Test database 4 - MongoDB",
"database": "testdb",
"type": "mongodb",
"username": "",
"password": "",
"port": 27017,
"host": "db-mongodb",
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 5 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data/workspace/data/app.db",
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 6 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data-2/workspace/data/app.db",
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 7 - Redis",
"type": "redis",
"port": 6379,
"host": "db-redis",
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 8 - Redis Auth",
"type": "redis",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-redis-auth",
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 9 - Valkey",
"type": "valkey",
"port": 6379,
"host": "db-valkey",
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 10 - Valkey Auth",
"type": "valkey",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-valkey-auth",
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Mysql",
"database": "mysqldb",
"type": "mysql",
"username": "mysqldb",
"password": "changeme",
"port": 3306,
"host": "db-mysql",
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 12 - Firebird",
"database": "/var/lib/firebird/data/mirror.fdb",
"type": "firebird",
"username": "alice",
"password": "fake_password",
"port": 3050,
"host": "db-firebird",
"generated_id": "16706124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 13 - MsSQL",
"database": "myappdb",
"type": "mssql",
"username": "sa",
"password": "Portabase!Strong1",
"port": 1433,
"host": "db-mssql",
"generated_id": "16706125-ff7e-4c97-8c83-0adeff214682"
},
{
"name": "Test database 14 - Docker Volume",
"type": "docker-volume",
"volume_name": "databases_sqlite-data",
"generated_id": "16706126-ff7e-4c97-8c83-0adeff214690",
"container_name": "db-sqlite"
}
]
}
+77 -3
View File
@@ -29,11 +29,85 @@ host = "db-mongodb-auth"
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 3 - MongoDB"
name = "Test database 4 - MongoDB"
database = "testdb"
type = "mongodb"
username = ""
password = ""
port = 27017
host = "db-mongodb"
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 5 - SQLite DB"
type = "sqlite"
path = "/sqlite-data/workspace/data/app.db"
generated_id = "16678178-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 6 - SQLite DB 2"
type = "sqlite"
path = "/sqlite-data-2/workspace/data/app.db"
generated_id = "16678179-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 7 - Redis"
type = "redis"
port = 6379
host = "db-redis"
generated_id = "16678166-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 8 - Redis Auth"
type = "redis"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-redis-auth"
generated_id = "16678160-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 9 - Valkey"
type = "valkey"
port = 6379
host = "db-valkey"
generated_id = "16678560-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 10 - Valkey Auth"
type = "valkey"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-valkey-auth"
generated_id = "16678561-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 11 - Mysql"
database = "mysqldb"
type = "mysql"
password = "changeme"
username = "mysqldb"
port = 3306
host = "db-mysql"
generated_id = "16656124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 12 - Firebird"
database = "/var/lib/firebird/data/mirror.fdb"
type = "firebird"
password = "fake_password"
username = "alice"
port = 3050
host = "db-firebird"
generated_id = "16706124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 13 - MSSQL"
database = "myappdb"
type = "mssql"
username = "sa"
password = "Portabase!Strong1"
port = 1433
host = "db-mssql"
generated_id = "16706125-ff7e-4c97-8c83-0adeff214682"
+213
View File
@@ -0,0 +1,213 @@
name: databases
services:
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
db-postgres-2:
container_name: db-postgres-2
image: postgres:17-alpine
ports:
- "5438:5432"
volumes:
- postgres-data-2:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb2
- POSTGRES_USER=devuser2
- POSTGRES_PASSWORD=changeme2
networks:
- portabase
db-mariadb:
container_name: db-mariadb
image: mariadb:latest
ports:
- "3311:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mariadb-data:/var/lib/mysql
networks:
- portabase
db-mysql:
container_name: db-mysql
image: mysql:9.5
ports:
- "3312:3306"
environment:
- MYSQL_DATABASE=mysqldb
- MYSQL_USER=mysqldb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mysql-data:/var/lib/mysql
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
environment:
MONGO_INITDB_DATABASE: testdb
networks:
- portabase
sqlite:
container_name: db-sqlite
image: keinos/sqlite3
volumes:
- sqlite-data:/workspace/data
working_dir: /workspace
command: tail -f /dev/null
stdin_open: true
tty: true
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "6378:6379"
volumes:
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "6380:6379"
volumes:
- redis-data-auth:/data
environment:
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
db-firebird:
image: firebirdsql/firebird
container_name: db-firebird
restart: always
environment:
- FIREBIRD_ROOT_PASSWORD=fake_root_password
- FIREBIRD_USER=alice
- FIREBIRD_PASSWORD=fake_password
- FIREBIRD_DATABASE=mirror.fdb
- FIREBIRD_DATABASE_DEFAULT_CHARSET=UTF8
volumes:
- firebird-data:/var/lib/firebird/data
ports:
- "3060:3050"
networks:
- portabase
db-mssql:
container_name: db-mssql
image: mcr.microsoft.com/azure-sql-edge:latest
ports:
- "1433:1433"
environment:
ACCEPT_EULA: "Y"
MSSQL_SA_PASSWORD: "Portabase!Strong1"
volumes:
- mssql-data:/var/opt/mssql
networks:
- portabase
healthcheck:
test: ["CMD-SHELL", "cat /proc/net/tcp6 | grep -q '059901' || exit 1"]
interval: 10s
timeout: 5s
retries: 20
volumes:
postgres-data:
postgres-data-2:
mariadb-data:
mysql-data:
mongodb-data:
mongodb-data-auth:
sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
firebird-data:
mssql-data:
networks:
portabase:
name: portabase_network
external: true
+2 -6
View File
@@ -5,23 +5,20 @@ services:
# dockerfile: docker/Dockerfile
# target: prod
image: portabase/agent:latest
# platform: linux/arm64
container_name: rust-prod
volumes:
- ./databases.json:/config/config.json
environment:
# APP_ENV: production
LOG: info
TZ: "Europe/Paris"
# DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
#DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "<to complete>"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
@@ -63,7 +60,6 @@ volumes:
mongodb-data:
mongodb-data-auth:
networks:
portabase:
name: portabase_network
+22
View File
@@ -0,0 +1,22 @@
services:
agent-test:
build:
context: .
dockerfile: docker/Dockerfile
target: dev
working_dir: /app
container_name: agent-test
volumes:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
TZ: Europe/Paris
EDGE_KEY: ""
volumes:
cargo-registry:
cargo-git:
+15 -76
View File
@@ -7,104 +7,43 @@ services:
container_name: rust-dev
volumes:
- .:/app
- ./databases.json:/config/config.json
# - ./databases.toml:/config/config.toml
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- ./databases.json:/config/config.json
#- ./databases.toml:/config/config.toml
- /var/run/docker.sock:/var/run/docker.sock
# - cargo-target:/app/target
- databases_sqlite-data:/sqlite-data/workspace/data
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
# - /bigdisk:/scratch
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiYTMxNWFhZmUtNDcyNi00OWM0LThhMmEtMGM2ZWQ0N2E1NjVkIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
# TMPDIR: /scratch
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNTljYzRjYTUtOTAyNy00ZThiLTk1NDktMjAzOTI3ZDVjNmUyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
cpus: "1.50"
mem_limit: 4g
memswap_limit: 4g
pids_limit: 512
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
db-mariadb:
container_name: db-mariadb
image: mariadb:latest
ports:
- "3311:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mariadb-data:/var/lib/mysql
networks:
- portabase
#
# db-mongodb-auth:
# container_name: db-mongodb-auth
# image: mongo:latest
# ports:
# - "27082:27017"
# environment:
# MONGO_INITDB_ROOT_USERNAME: root
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
# MONGO_INITDB_DATABASE: testdbauth
# command: mongod --auth
# networks:
# - portabase
# volumes:
# - mongodb-data-auth:/data/db
# healthcheck:
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
#
# db-mongodb:
# container_name: db-mongodb
# image: mongo:latest
# ports:
# - "27083:27017"
# volumes:
# - mongodb-data:/data/db
# healthcheck:
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
# environment:
# MONGO_INITDB_DATABASE: testdb
# networks:
# - portabase
volumes:
cargo-registry:
cargo-git:
# cargo-target:
postgres-data:
mariadb-data:
# mongodb-data:
# mongodb-data-auth:
databases_sqlite-data:
external: true
networks:
portabase:
name: portabase_network
external: true
# docker network create portabase_network
+51 -2
View File
@@ -1,7 +1,15 @@
# =========================
# MySQL client tools
# =========================
FROM mysql:8.4 AS mysql-client-tools
RUN mkdir -p /mysql-exports/bin /mysql-exports/lib \
&& cp /usr/bin/mysqldump /mysql-exports/bin/ \
&& find /usr/lib -name "libmysqlclient.so.21*" -exec cp {} /mysql-exports/lib/ \;
# =========================
# Base image (shared)
# =========================
FROM rust:1.92.0 AS base
FROM rust:1.94.0 AS base
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config \
@@ -15,9 +23,27 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
zlib1g \
curl \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
&& /tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/dotnet \
&& rm /tmp/dotnet-install.sh \
&& /usr/local/dotnet/dotnet tool install --global microsoft.sqlpackage
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& curl -sSL "https://github.com/microsoft/go-sqlcmd/releases/latest/download/sqlcmd-linux-${ARCH}.tar.bz2" \
| tar -xjf - -C /usr/local/bin sqlcmd \
&& chmod +x /usr/local/bin/sqlcmd
ARG TARGETARCH
# =========================
@@ -43,12 +69,20 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
chmod +x /usr/lib/postgresql/*/bin/*
# =========================
# MongoDB client binaries
# =========================
COPY assets/tools/${TARGETARCH}/mongodb/ /usr/local/mongodb/
RUN chmod +x /usr/local/mongodb/bin/*
# =========================
# MySQL real mysqldump binary
# =========================
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
WORKDIR /app
# =========================
@@ -57,6 +91,7 @@ WORKDIR /app
FROM base AS dev
RUN cargo install cargo-watch
RUN cargo install grcov --locked
COPY Cargo.toml Cargo.lock ./
RUN mkdir src && echo "fn main() {}" > src/main.rs
@@ -91,9 +126,19 @@ RUN apt-get update && apt-get install -y \
libreadline8 \
libncurses6 \
zlib1g \
curl \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& rm -rf /var/lib/apt/lists/*
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
&& /tmp/dotnet-install.sh --channel 8.0 --runtime dotnet --install-dir /usr/local/dotnet \
&& rm /tmp/dotnet-install.sh
WORKDIR /app
@@ -104,8 +149,12 @@ RUN chmod +x /entrypoint.sh
COPY --from=base /usr/lib/postgresql/ /usr/lib/postgresql/
COPY --from=base /usr/local/mongodb/bin/ /usr/local/mongodb/bin/
COPY --from=base /root/.dotnet/tools/ /root/.dotnet/tools/
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
ENV APP_ENV=production
CMD ["/entrypoint.sh"]
+44
View File
@@ -0,0 +1,44 @@
#!/bin/bash
set -euo pipefail
check_docker() {
if ! docker info > /dev/null 2>&1; then
echo "Docker is not running. Attempting to start Docker..."
if [[ "$OSTYPE" == "darwin"* ]]; then
open -a Docker
echo "Waiting for Docker to start..."
until docker info > /dev/null 2>&1; do
sleep 2
done
elif command -v systemctl >/dev/null 2>&1; then
sudo systemctl start docker
else
echo "Cannot start Docker automatically. Please start Docker manually."
exit 1
fi
else
echo "Docker is running."
fi
}
check_network() {
local network_name="portabase_network"
if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then
echo "Docker network '${network_name}' not found. Creating..."
docker network create "${network_name}"
else
echo "Docker network '${network_name}' already exists."
fi
}
check_docker
check_network
echo "Stopping old database containers..."
docker compose -f ./docker-compose.databases.yml down
echo "Starting database containers..."
docker compose -f ./docker-compose.databases.yml up -d
echo "Starting main services..."
docker compose -f ./docker-compose.yml up
+10 -3
View File
@@ -34,16 +34,23 @@ if [ -n "$TZ" ]; then
fi
fi
REDIS_PORT=65515
echo "[entrypoint] APP_ENV=$APP_ENV"
echo "[entrypoint] Starting Redis..."
redis-server --daemonize yes
redis-server --port $REDIS_PORT --daemonize yes
echo "[entrypoint] Waiting for Redis to be ready..."
until redis-cli ping >/dev/null 2>&1; do
MAX_RETRIES=20
COUNT=0
until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do
COUNT=$((COUNT+1))
if [ $COUNT -ge $MAX_RETRIES ]; then
echo "[ERROR] Redis did not start after $MAX_RETRIES attempts"
exit 1
fi
echo "[entrypoint] Redis not ready, sleeping 1s..."
sleep 1
done
echo "[entrypoint] Redis is ready"
+17
View File
@@ -0,0 +1,17 @@
.DS_Store
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
*.swp
*.bak
*.tmp
*.orig
*~
.project
.idea/
*.tmproj
.vscode/
+29
View File
@@ -0,0 +1,29 @@
apiVersion: v2
name: portabase-agent
description: Helm chart for Portabase Agent
type: application
version: 0.0.0
appVersion: "latest"
keywords:
- postgresql
- mariadb
- mongodb
- mysql
- sqlite
- backup
- database
- restore
- agent
home: https://github.com/Portabase/agent
sources:
- https://github.com/Portabase/agent
- https://github.com/Portabase/agent/tree/main/helm
maintainers:
- name: Charles Gauthereau
url: https://github.com/RambokDev
- name: Killian Larcher
url: https://github.com/KillianLarcher
icon: https://raw.githubusercontent.com/Portabase/agent/main/.github/assets/logo.png
+59
View File
@@ -0,0 +1,59 @@
# Development Notes
## Check that Kubernetes is reachable locally
```bash
kubectl get nodes
```
## Install the local Portabase Agent Helm chart
```bash
helm install portabase-agent . \
--set env.EDGE_KEY=<your-edge-key>
```
## Check the pods
```bash
kubectl get pods
```
## Check the services
```bash
kubectl get svc
```
## To update .env variables or JSON config:
```bash
kubectl rollout restart deployment portabase-agent
```
## Install or upgrade the Helm chart
```bash
helm upgrade portabase-agent . \
--reuse-values \
--set env.EDGE_KEY="NEW_EDGE_KEY"
```
## Rollout to restart
```bash
kubectl rollout restart deployment portabase-agent
```
## List pods to get the pod name
```bash
kubectl get pods -l app=portabase-agent
```
## Get logs for the pod
```bash
kubectl logs portabase-agent-6f7d4f5c6b-abc12
```
## Uninstall Agent
``` bash
helm uninstall portabase-agent
```
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: portabase-agent-config
data:
config.json: |
{{ .Values.volume.configFile.content | nindent 4 }}
+62
View File
@@ -0,0 +1,62 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: portabase-agent
labels:
app: portabase-agent
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: portabase-agent
template:
metadata:
labels:
app: portabase-agent
spec:
hostAliases:
{{- range .Values.network.hostAliases }}
- ip: {{ .ip }}
hostnames:
{{- range .hostnames }}
- {{ . }}
{{- end }}
{{- end }}
containers:
- name: portabase-agent
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
envFrom:
- configMapRef:
name: portabase-agent-env
volumeMounts:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
- name: config
mountPath: /config/config.json
subPath: config.json
readOnly: true
# uses hostPath
{{- else }}
- name: config
mountPath: /config/config.json
subPath: config.json
# uses ConfigMap content
{{- end }}
{{- end }}
volumes:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
- name: config
hostPath:
path: {{ .Values.volume.configFile.hostPath }}
type: File
{{- else }}
- name: config
configMap:
name: portabase-agent-config
items:
- key: config.json
path: config.json
{{- end }}
{{- end }}
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: portabase-agent-env
data:
EDGE_KEY: {{ .Values.env.EDGE_KEY | quote }}
TZ: {{ .Values.env.TZ | quote }}
POLLING: {{ .Values.env.POLLING | quote }}
APP_ENV: {{ .Values.env.APP_ENV | quote }}
LOG: {{ .Values.env.LOG | quote }}
+48
View File
@@ -0,0 +1,48 @@
replicaCount: 1
image:
repository: portabase/agent
tag: latest
pullPolicy: IfNotPresent
env:
EDGE_KEY: "your_edge_key_here"
TZ: "UTC"
POLLING: "5"
APP_ENV: "production"
LOG: "info"
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 250m
memory: 256Mi
volume:
configFile:
enabled: true
hostPath: "" # Use host file if set, otherwise use `content`
content: | # JSON content for config.json if no hostPath
{
"databases": [
{
"name": "my-site-prod (readable name)",
"database": "devdb",
"type": "postgresql",
"host": "localhost",
"port": 5432,
"username": "admin_prod",
"password": "super_secure_password",
"generated_id": "550e8400-e29b-41d4-a716-446655440000"
}
]
}
network:
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "localhost"
+71
View File
@@ -0,0 +1,71 @@
set dotenv-load := true
set shell := ["bash", "-cu"]
CLUSTER_SCRIPT := "docker/entrypoints/app-dev-entrypoint.sh"
up:
bash {{CLUSTER_SCRIPT}}
seed-mongo:
echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
echo "Seeding MySQL (1GB)..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed.sql
seed-postgres-1gb:
echo "Seeding Postgres (1GB)..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed-1gb.sql
seed-sqlite:
echo "Seeding SQLite..."
bash ./scripts/sqlite/seed.sh
echo "Done"
seed-firebird:
echo "Seeding Firebird..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed.sql
echo "Verifying Firebird tables..."
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
seed-mssql:
echo "Seeding MSSQL..."
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
echo "Done"
seed-all:
just seed-mongo
just seed-mysql
just seed-postgres
just seed-postgres-1gb
just seed-sqlite
just seed-mongo
just seed-firebird
just seed-mssql
test:
echo "Build test image (with grcov included)"
docker compose -f docker-compose.test.yml build agent-test
echo "Start agent-test container"
docker compose -f docker-compose.test.yml up -d agent-test
echo "Run tests inside container"
docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync"
echo "Down volumes tests databases"
docker compose -f docker-compose.test.yml down --volumes
-99
View File
@@ -1,99 +0,0 @@
#!/bin/bash
set -e
if [ -z "$1" ]; then
echo "Usage: ./release <version>"
echo "Example: ./release v1.0.0"
exit 1
fi
VERSION=$1
CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
if [ "$CURRENT_BRANCH" = "main" ]; then
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "Error: On 'main' branch, only release tags (X.Y.Z) are allowed."
exit 1
fi
else
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then
echo "Error: On branch '$CURRENT_BRANCH', only RC tags matching X.Y.Z-rc.W are allowed (e.g., 1.0.0-rc.1)."
exit 1
fi
fi
CLEAN_VERSION=${VERSION#v}
CURRENT_DATE=$(date +%Y-%m-%d)
echo "Preparing release $VERSION..."
# package.json
if [ -f package.json ]; then
echo "Updating package.json..."
if sed --version >/dev/null 2>&1; then
sed -i "s/\"version\": \".*\"/\"version\": \"$CLEAN_VERSION\"/" package.json
else
sed -i '' "s/\"version\": \".*\"/\"version\": \"$CLEAN_VERSION\"/" package.json
fi
fi
# pyproject.toml
if [ -f pyproject.toml ]; then
echo "Updating pyproject.toml..."
if sed --version >/dev/null 2>&1; then
sed -i "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" pyproject.toml
else
sed -i '' "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" pyproject.toml
fi
fi
# Cargo.toml
if [ -f Cargo.toml ]; then
echo "Updating Cargo.toml..."
if sed --version >/dev/null 2>&1; then
sed -i "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" Cargo.toml
else
sed -i '' "s/^version = \".*\"/version = \"$CLEAN_VERSION\"/" Cargo.toml
fi
fi
# CITATION.cff
if [ -f CITATION.cff ]; then
echo "Updating CITATION.cff..."
if sed --version >/dev/null 2>&1; then
sed -i "s/^version: .*/version: $CLEAN_VERSION/" CITATION.cff
sed -i "s/^date-released: .*/date-released: \"$CURRENT_DATE\"/" CITATION.cff
else
sed -i '' "s/^version: .*/version: $CLEAN_VERSION/" CITATION.cff
sed -i '' "s/^date-released: .*/date-released: \"$CURRENT_DATE\"/" CITATION.cff
fi
fi
git add .
if ! git diff-index --quiet HEAD --; then
echo "Committing changes..."
git commit -m "chore(release): $VERSION"
else
echo "No changes to commit. Proceeding to tag..."
fi
if git rev-parse "$VERSION" >/dev/null 2>&1; then
echo "Tag $VERSION already exists. Aborting."
exit 1
fi
echo "Creating tag $VERSION..."
git tag -a "$VERSION" -m "Release $VERSION"
echo "Pushing changes and tags to remote..."
git push
git push origin "$VERSION"
echo "Successfully released $VERSION!"
+20 -2
View File
@@ -1,12 +1,30 @@
# Seed instructions
## MongoDB
```bash
make seed-mongo
make seed-mongo-auth
make seed-mysql
make seed-mysql-1gb
make seed-postgres
make seed-postgres-1gb
make seed-all
make seed-all
make seed-sqlite
make seed-sqlite SEED=big
```
## Verify commands
### Sqlite
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LIMIT 10;"
```
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
```
```bash
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
```
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE users (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
email VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
COMMIT;
+20
View File
@@ -0,0 +1,20 @@
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'myappdb')
CREATE DATABASE [myappdb];
GO
USE [myappdb];
GO
IF OBJECT_ID('users', 'U') IS NULL
CREATE TABLE users (
id INT IDENTITY(1,1) PRIMARY KEY,
email NVARCHAR(255) NOT NULL UNIQUE,
name NVARCHAR(255),
created_at DATETIME DEFAULT GETDATE()
);
GO
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
INSERT INTO users (email, name) VALUES ('charlie@example.com', 'Charlie');
GO
+1 -1
View File
@@ -60,7 +60,7 @@ SELECT
)
FROM users u
JOIN generate_series(1, 30) AS p(post_no)
ON u.id <= 300000;
ON u.id <= 500000;
-- ============================================================
-- OPTIONAL: FORCE DISK MATERIALIZATION
+74
View File
@@ -0,0 +1,74 @@
PRAGMA foreign_keys = ON;
BEGIN TRANSACTION;
-- Drop tables
DROP TABLE IF EXISTS users;
DROP TABLE IF EXISTS projects;
DROP TABLE IF EXISTS tasks;
-- Users
CREATE TABLE users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
email TEXT NOT NULL UNIQUE,
full_name TEXT NOT NULL,
role TEXT NOT NULL CHECK (role IN ('admin','manager','user')),
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
-- Projects
CREATE TABLE projects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
owner_id INTEGER NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE
);
-- Tasks
CREATE TABLE tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
project_id INTEGER NOT NULL,
title TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')),
priority INTEGER NOT NULL DEFAULT 3,
due_date TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE
);
-- Seed minimal users
INSERT INTO users (email, full_name, role) VALUES
('admin@example.com', 'System Admin', 'admin'),
('manager@example.com', 'Project Manager', 'manager'),
('user@example.com', 'Standard User', 'user');
-- Generate 10_000 projects
WITH RECURSIVE numbers(x) AS (
SELECT 1
UNION ALL
SELECT x+1 FROM numbers WHERE x<10000
)
INSERT INTO projects (name, description, owner_id)
SELECT
'Project #' || x,
'Auto-generated project description for project #' || x,
(1 + (x % 3)) -- cycle users 1..3
FROM numbers;
-- Generate 100_000 tasks
WITH RECURSIVE numbers(x) AS (
SELECT 1
UNION ALL
SELECT x+1 FROM numbers WHERE x<100000
)
INSERT INTO tasks (project_id, title, status, priority, due_date)
SELECT
(1 + (x % 10000)), -- project id cycle
'Task #' || x,
CASE (x % 3) WHEN 0 THEN 'todo' WHEN 1 THEN 'in_progress' ELSE 'done' END,
1 + (x % 5),
date('now', '+' || (x % 30) || ' days')
FROM numbers;
COMMIT;
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
SEED_VALUE="${SEED:-}"
if [ "$SEED_VALUE" = "big" ]; then
SQLITE_SEED_FILE="./scripts/sqlite/seed-big.sql"
else
SQLITE_SEED_FILE="./scripts/sqlite/seed.sql"
fi
docker exec -u 0 db-sqlite sh -c "chmod -R 777 /workspace/data"
docker exec -u 0 db-sqlite sh -c "touch /workspace/data/app.db"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < "$SQLITE_SEED_FILE"
+59
View File
@@ -0,0 +1,59 @@
PRAGMA foreign_keys = ON;
BEGIN TRANSACTION;
-- Drop existing tables (idempotent reset)
DROP TABLE IF EXISTS users;
DROP TABLE IF EXISTS projects;
DROP TABLE IF EXISTS tasks;
-- Users
CREATE TABLE users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
email TEXT NOT NULL UNIQUE,
full_name TEXT NOT NULL,
role TEXT NOT NULL CHECK (role IN ('admin','manager','user')),
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
-- Projects
CREATE TABLE projects (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
owner_id INTEGER NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE
);
-- Tasks
CREATE TABLE tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
project_id INTEGER NOT NULL,
title TEXT NOT NULL,
status TEXT NOT NULL CHECK (status IN ('todo','in_progress','done')),
priority INTEGER NOT NULL DEFAULT 3,
due_date TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE
);
-- Seed users
INSERT INTO users (email, full_name, role) VALUES
('admin@example.com', 'System Admin', 'admin'),
('manager@example.com', 'Project Manager', 'manager'),
('user@example.com', 'Standard User', 'user');
-- Seed projects
INSERT INTO projects (name, description, owner_id) VALUES
('Internal Tooling', 'Backoffice automation platform', 2),
('Client Portal', 'Customer-facing SaaS interface', 2);
-- Seed tasks
INSERT INTO tasks (project_id, title, status, priority, due_date) VALUES
(1, 'Define architecture', 'done', 1, date('now', '+3 days')),
(1, 'Implement authentication', 'in_progress', 1, date('now', '+7 days')),
(2, 'Design landing page', 'todo', 2, date('now', '+5 days')),
(2, 'Setup CI/CD', 'todo', 2, date('now', '+10 days'));
COMMIT;
Binary file not shown.
+16 -8
View File
@@ -4,11 +4,11 @@ use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::cron::CronService;
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::restore::RestoreService;
pub struct Agent {
ctx: Arc<Context>,
@@ -43,22 +43,30 @@ impl Agent {
let ping_result = self.status_service.ping(&config.databases).await?;
for db in ping_result.databases.iter() {
let database = config
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
.unwrap();
info!(
"Generated Id: {} | backup action: {} | restore action: {}",
db.generated_id, db.data.backup.action, db.data.restore.action
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
);
let _ = self.cron_service.sync(db).await;
if db.data.backup.action {
let _ = self
.backup_service
.dispatch(&db.generated_id, &config, method.clone())
.dispatch(
&db.generated_id,
&config,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self
.restore_service
.dispatch(db, &config)
.await;
let _ = self.restore_service.dispatch(db, &config).await;
}
}
+9 -1
View File
@@ -1,3 +1,4 @@
use crate::services::api::ApiClient;
use crate::settings::CONFIG;
use crate::utils::edge_key::{EdgeKey, EdgeKeyError, decode_edge_key};
use tracing::{debug, error, info};
@@ -6,6 +7,7 @@ use tracing::{debug, error, info};
pub struct Context {
#[allow(dead_code)]
pub edge_key: EdgeKey,
pub api: ApiClient,
}
impl Context {
@@ -34,6 +36,12 @@ impl Context {
}
};
Context { edge_key }
let server_url = format!("{}/api", edge_key.server_url);
let api_client = ApiClient::new(server_url);
Context {
edge_key: edge_key,
api: api_client,
}
}
}
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod context;
pub mod agent;
pub mod context;
+68
View File
@@ -0,0 +1,68 @@
use crate::domain::docker_volume::docker::{
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use bollard::query_parameters::DownloadFromContainerOptions;
use futures_util::StreamExt;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
pub async fn run(cfg: DatabaseConfig, backup_dir: PathBuf, logger: Arc<JobLogger>) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
futures::executor::block_on(async move {
logger.log("info", format!("Starting docker-volume backup for {}", cfg.name));
let docker = client()?;
let image = resolve_helper_image(&docker).await?;
logger.log("debug", format!("Helper image: {image}"));
if let Some(name) = &cfg.container_name {
logger.log("info", format!("Stopping container {name} for consistent backup"));
stop_container(&docker, name).await?;
}
let result = async {
let helper = create_helper(&docker, &image, &cfg.volume_name, &cfg.generated_id, true, None).await?;
let file_path = backup_dir.join(format!("{}.tar", cfg.generated_id));
let start = Instant::now();
let dl_opts = DownloadFromContainerOptions { path: "/vol".to_string() };
let mut stream = docker.download_from_container(&helper.id, Some(dl_opts));
let mut out = File::create(&file_path)
.await
.with_context(|| format!("Failed to create backup file {}", file_path.display()))?;
let mut bytes_written: u64 = 0;
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("Error streaming volume archive from Docker")?;
bytes_written += chunk.len() as u64;
out.write_all(&chunk).await?;
}
out.flush().await?;
let duration_ms = start.elapsed().as_millis() as f64;
logger.log_command("docker download_from_container", None, Some(0), Some(duration_ms));
logger.log("info", format!("Volume backup wrote {bytes_written} bytes to {}", file_path.display()));
remove_helper(&docker, &helper.id).await;
anyhow::Ok(file_path)
}
.await;
if let Some(name) = &cfg.container_name {
if let Err(e) = start_container(&docker, name).await {
logger.log("error", format!("Failed to restart container {name}: {e}"));
}
}
result
})
})
.await?
}
+45
View File
@@ -0,0 +1,45 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
pub struct DockerVolumeDatabase {
cfg: DatabaseConfig,
}
impl DockerVolumeDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for DockerVolumeDatabase {
fn file_extension(&self) -> &'static str {
".tar"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+169
View File
@@ -0,0 +1,169 @@
#![allow(dead_code)]
use anyhow::{Context, Result};
use bollard::Docker;
use bollard::models::{ContainerCreateBody, HostConfig};
use bollard::query_parameters::{
CreateContainerOptions, InspectContainerOptions, ListContainersOptions,
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
};
use std::collections::HashMap;
use tracing::{info, warn};
use uuid::Uuid;
pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
const HELPER_MOUNT: &str = "/vol";
pub fn client() -> Result<Docker> {
Docker::connect_with_unix_defaults().context("Failed to connect to Docker daemon socket")
}
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
for src in [mountinfo, cgroup] {
for line in src.lines() {
for marker in ["/containers/", "/docker/"] {
if let Some(idx) = line.find(marker) {
let rest = &line[idx + marker.len()..];
let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
if id.len() >= 64 {
return Some(id[..64].to_string());
}
}
}
}
}
None
}
pub async fn resolve_helper_image(docker: &Docker) -> Result<String> {
if let Ok(img) = std::env::var("PORTABASE_HELPER_IMAGE") {
if !img.trim().is_empty() {
return Ok(img);
}
}
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
let cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default();
let id = parse_container_id(&mountinfo, &cgroup).context(
"Could not determine own container id; set PORTABASE_HELPER_IMAGE to a locally-present image",
)?;
let info = docker
.inspect_container(&id, None::<InspectContainerOptions>)
.await
.with_context(|| format!("Failed to inspect self container {id}"))?;
info.image
.context("Self container inspection returned no image reference")
}
pub struct Helper {
pub id: String,
}
pub async fn create_helper(
docker: &Docker,
image: &str,
volume_name: &str,
generated_id: &str,
read_only: bool,
cmd: Option<Vec<String>>,
) -> Result<Helper> {
let bind = format!(
"{volume_name}:{HELPER_MOUNT}{}",
if read_only { ":ro" } else { "" }
);
let mut labels = HashMap::new();
labels.insert(EPHEMERAL_LABEL.to_string(), "true".to_string());
labels.insert("com.docker.compose.project".to_string(), String::new());
labels.insert("com.docker.compose.service".to_string(), String::new());
labels.insert("com.docker.compose.oneoff".to_string(), String::new());
let name = format!(
"portabase-vol-{generated_id}-{}",
&Uuid::new_v4().to_string()[..8]
);
let body = ContainerCreateBody {
image: Some(image.to_string()),
cmd,
labels: Some(labels),
host_config: Some(HostConfig {
binds: Some(vec![bind]),
auto_remove: Some(false),
..Default::default()
}),
..Default::default()
};
let opts = CreateContainerOptions {
name: Some(name),
..Default::default()
};
let res = docker
.create_container(Some(opts), body)
.await
.with_context(|| format!("Failed to create helper container for volume {volume_name}"))?;
Ok(Helper { id: res.id })
}
pub async fn remove_helper(docker: &Docker, id: &str) {
let stop_opts = StopContainerOptions {
t: Some(2),
..Default::default()
};
let _ = docker.stop_container(id, Some(stop_opts)).await;
if let Ok(info) = docker
.inspect_container(id, None::<InspectContainerOptions>)
.await
{
let name = info.name.unwrap_or_default();
let name = name.trim_start_matches('/');
let code = info.state.and_then(|s| s.exit_code).unwrap_or_default();
info!("Helper container {name} exited with code {code}");
}
let opts = RemoveContainerOptions {
force: true,
..Default::default()
};
if let Err(e) = docker.remove_container(id, Some(opts)).await {
warn!("Failed to remove helper container {id}: {e}");
}
}
pub async fn stop_container(docker: &Docker, name: &str) -> Result<()> {
docker
.stop_container(name, None::<StopContainerOptions>)
.await
.with_context(|| format!("Failed to stop container {name}"))
}
pub async fn start_container(docker: &Docker, name: &str) -> Result<()> {
docker
.start_container(name, None::<StartContainerOptions>)
.await
.with_context(|| format!("Failed to start container {name}"))
}
pub async fn sweep_ephemeral(docker: &Docker) -> Result<usize> {
let mut filters = HashMap::new();
filters.insert("label".to_string(), vec![format!("{EPHEMERAL_LABEL}=true")]);
let opts = ListContainersOptions {
all: true,
filters: Some(filters),
..Default::default()
};
let list = docker.list_containers(Some(opts)).await?;
let mut removed = 0;
for c in list {
if let Some(id) = c.id {
remove_helper(docker, &id).await;
removed += 1;
}
}
Ok(removed)
}
+5
View File
@@ -0,0 +1,5 @@
pub mod backup;
pub mod database;
pub mod docker;
pub mod ping;
pub mod restore;
+12
View File
@@ -0,0 +1,12 @@
use crate::domain::docker_volume::docker::client;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let docker = client()?;
match docker.inspect_volume(&cfg.volume_name).await {
Ok(_) => Ok(true),
Err(bollard::errors::Error::DockerResponseServerError { status_code: 404, .. }) => Ok(false),
Err(e) => Err(e.into()),
}
}
+102
View File
@@ -0,0 +1,102 @@
use crate::domain::docker_volume::docker::{
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use bollard::exec::StartExecResults;
use bollard::models::ExecConfig;
use bollard::query_parameters::UploadToContainerOptions;
use futures_util::StreamExt;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
futures::executor::block_on(async move {
logger.log("info", format!("Starting docker-volume restore for {}", cfg.name));
let docker = client()?;
let image = resolve_helper_image(&docker).await?;
logger.log("debug", format!("Restore archive: {}", archive.display()));
if let Some(name) = &cfg.container_name {
logger.log("info", format!("Stopping container {name} for restore"));
stop_container(&docker, name).await?;
}
let result = async {
let helper = create_helper(
&docker,
&image,
&cfg.volume_name,
&cfg.generated_id,
false,
Some(vec![
"sh".into(),
"-c".into(),
"trap 'exit 0' TERM; sleep 2147483647 & wait".into(),
]),
)
.await?;
start_container(&docker, &helper.id).await?;
let exec = docker
.create_exec(
&helper.id,
ExecConfig {
cmd: Some(vec![
"sh".to_string(),
"-c".to_string(),
"rm -rf /vol/* /vol/.[!.]* 2>/dev/null || true".to_string(),
]),
attach_stdout: Some(true),
attach_stderr: Some(true),
..Default::default()
},
)
.await
.context("Failed to create wipe exec")?;
if let StartExecResults::Attached { mut output, .. } =
docker.start_exec(&exec.id, None).await.context("Failed to run wipe exec")?
{
while output.next().await.is_some() {}
}
let start = Instant::now();
let file = tokio::fs::File::open(&archive)
.await
.with_context(|| format!("Failed to open {}", archive.display()))?;
let stream = tokio_util::io::ReaderStream::new(file);
let up_opts = UploadToContainerOptions { path: "/".to_string(), ..Default::default() };
docker
.upload_to_container(&helper.id, Some(up_opts), bollard::body_try_stream(stream))
.await
.context("Failed to upload volume archive")?;
let duration_ms = start.elapsed().as_millis() as f64;
logger.log_command("docker upload_to_container", None, Some(0), Some(duration_ms));
remove_helper(&docker, &helper.id).await;
logger.log("info", format!("Volume restore completed for {}", cfg.name));
anyhow::Ok(())
}
.await;
if let Some(name) = &cfg.container_name {
if let Err(e) = start_container(&docker, name).await {
logger.log("error", format!("Failed to restart container {name}: {e}"));
}
}
result
})
})
.await?
}
+27 -4
View File
@@ -1,7 +1,16 @@
use crate::domain::docker_volume::database::DockerVolumeDatabase;
use crate::domain::mongodb::database::MongoDatabase;
use crate::domain::mysql::database::MySQLDatabase;
use crate::domain::postgres::cluster::database::PostgresClusterDatabase;
use crate::domain::postgres::database::PostgresDatabase;
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
use crate::domain::redis::database::RedisDatabase;
use crate::domain::sqlite::database::SqliteDatabase;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::domain::firebird::database::FirebirdDatabase;
use crate::domain::mariadb::database::MariaDBDatabase;
use crate::domain::mssql::database::MssqlDatabase;
use crate::services::backup::logger::JobLogger;
use crate::services::config::{DatabaseConfig, DbType};
use anyhow::Result;
use std::path::{Path, PathBuf};
@@ -11,8 +20,8 @@ use std::sync::Arc;
pub trait Database: Send + Sync {
fn file_extension(&self) -> &'static str;
async fn ping(&self) -> Result<bool>;
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path) -> Result<()>;
async fn backup(&self, backup_dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path, logger: Arc<JobLogger>) -> Result<()>;
}
pub struct DatabaseFactory;
@@ -24,9 +33,16 @@ impl DatabaseFactory {
let format = detect_format_from_size(&cfg).await;
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
}
}
@@ -36,9 +52,16 @@ impl DatabaseFactory {
let format = detect_format_from_file(restore_file);
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
}
}
}
+54
View File
@@ -0,0 +1,54 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting Firebird backup for database: {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
logger.log("info", format!("Firebird target: {}{}", db_path, file_path.display()));
let start = Instant::now();
let output = Command::new("gbak")
.arg("-b")
.arg("-v")
.arg("-user").arg(&cfg.username)
.arg("-password").arg(&cfg.password)
.arg(&db_path)
.arg(&file_path)
.output()
.with_context(|| format!("Failed to run gbak for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let combined_output = if stderr.is_empty() && stdout.is_empty() {
None
} else {
Some(format!("{}{}", stdout, stderr).trim().to_string())
};
if !output.status.success() {
logger.log_command("gbak", combined_output, Some(exit_code), Some(duration_ms));
anyhow::bail!("Firebird backup failed for {}: {}", cfg.name, stderr);
}
logger.log_command("gbak", combined_output, Some(0), Some(duration_ms));
logger.log("info", format!("Firebird backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+51
View File
@@ -0,0 +1,51 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct FirebirdDatabase {
cfg: DatabaseConfig,
}
impl FirebirdDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for FirebirdDatabase {
fn file_extension(&self) -> &'static str {
".fbk"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+4
View File
@@ -0,0 +1,4 @@
pub mod database;
mod ping;
mod backup;
mod restore;
+57
View File
@@ -0,0 +1,57 @@
use std::process::Stdio;
use tracing::{error, info};
use tokio::io::AsyncWriteExt;
use crate::services::config::DatabaseConfig;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
info!("Running Ping database from {}", db_path);
let mut child = Command::new("isql-fb")
.arg("-q")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&db_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
if let Some(mut stdin) = child.stdin.take() {
stdin.write_all(query).await?;
}
let output = match timeout(Duration::from_secs(5), child.wait_with_output()).await {
Ok(res) => res?,
Err(_) => return Ok(false),
};
if !output.status.success() {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if stderr.to_lowercase().contains("error") {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
if stdout.contains("1") {
return Ok(true);
}
info!("stdout {}", stdout);
error!("stderr {}", stderr);
Ok(false)
}
+48
View File
@@ -0,0 +1,48 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("debug", format!("Starting Firebird restore for database {}", cfg.name));
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
logger.log("info", format!("Restore source: {}", restore_file.display()));
logger.log("info", format!("Restore target: {}", db_path));
let start = Instant::now();
let output = Command::new("gbak")
.arg("-c")
.arg("-v")
.arg("-replace_database")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&restore_file)
.arg(&db_path)
.output()
.with_context(|| format!("Failed to run gbak restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
logger.log_command("gbak", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("Firebird restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("Firebird restore failed for {}: {}", cfg.name, stderr);
}
logger.log_command("gbak", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Firebird restore completed for {}", cfg.name));
Ok(())
})
.await?
}
+81
View File
@@ -0,0 +1,81 @@
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
logger.log("info", format!("MariaDB version detected: {}", v));
v
}
Err(e) => {
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let _mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
logger.log("debug", format!("Using mariadb-dump at {}", _mariadb_dump.display()));
if let Ok(out) = Command::new("mariadb-dump").arg("--version").output() {
logger.log("debug", format!("mariadb-dump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
}
logger.log("info", format!("Running mariadb-dump for {}", cfg.name));
let start = Instant::now();
let output = Command::new("mariadb-dump")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--single-transaction")
.arg("--quick")
.arg("--skip-lock-tables")
.arg("--no-create-db")
.arg("--skip-add-drop-table")
.arg("--compress")
.arg("--verbose")
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
.arg("--net-buffer-length=16K")
.arg("--default-character-set=utf8mb4")
.arg(&cfg.database)
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log_command("mariadb-dump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
}
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log_command("mariadb-dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("mariadb-dump completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+45
View File
@@ -0,0 +1,45 @@
use std::path::PathBuf;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("Version query failed: {}", stderr);
}
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
pub fn select_mariadb_path(version: &str) -> PathBuf {
let mut parts = version.split('.');
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
if major < 10 || (major == 10 && minor <= 6) {
"/usr/local/mariadb-10.6/bin".into()
} else {
"/usr/local/mariadb-12.1/bin".into()
}
}
+58
View File
@@ -0,0 +1,58 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MariaDBDatabase {
cfg: DatabaseConfig,
}
impl MariaDBDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
impl Database for MariaDBDatabase {
fn file_extension(&self) -> &'static str {
".sql"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod ping;
mod restore;
+28
View File
@@ -0,0 +1,28 @@
use crate::services::config::DatabaseConfig;
use std::collections::HashMap;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.arg("ping")
.envs(env);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output?;
Ok(output.status.success())
}
Err(_) => Ok(false),
}
}
+91
View File
@@ -0,0 +1,91 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::fs::File;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting restore for database {}", cfg.name));
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
cfg.database
);
let drop_start = Instant::now();
let drop_output = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.output()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
if !drop_output.status.success() {
logger.log_command("mariadb", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
logger.log_command("mariadb", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
let start = Instant::now();
let mut child = Command::new("mariadb")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("--database")
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
std::io::copy(&mut file, &mut stdin)
.context("Failed to stream SQL content to MariaDB stdin")?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
logger.log_command("mariadb", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MariaDB restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
}
logger.log_command("mariadb", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
Ok(())
});
handle.await??;
Ok(())
}
+9 -3
View File
@@ -1,5 +1,11 @@
pub mod docker_volume;
pub mod factory;
pub mod postgres;
pub mod mysql;
mod mongodb;
pub mod mysql;
pub mod postgres;
mod redis;
mod sqlite;
mod valkey;
mod mariadb;
mod firebird;
pub mod mssql;
+19 -6
View File
@@ -1,35 +1,48 @@
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting MongoDB backup for database {}", cfg.name);
logger.log("info", format!("Starting MongoDB backup for database {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mongodump = select_mongo_path().join("mongodump");
let uri = get_mongo_uri(cfg.clone());
let uri = get_mongo_uri(cfg.clone())?;
logger.log("info", format!("Running mongodump for {}", cfg.name));
let start = Instant::now();
let output = Command::new(mongodump)
.arg(format!("--uri={}", uri))
.arg(format!("--archive={}", file_path.display()))
.arg("--gzip")
.arg("--verbose")
.output()
.context("MongoDB backup failed")?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MongoDB backup failed for {}: {}", cfg.name, stderr);
logger.log("error", format!("MongoDB backup failed for {}: {}", cfg.name, stderr));
logger.log_command("mongodump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("MongoDB backup failed for {}: {}", cfg.name, stderr);
}
info!("MongoDB backup completed for {}", cfg.name);
logger.log_command("mongodump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("MongoDB backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
+24 -7
View File
@@ -3,7 +3,7 @@ use anyhow::Result;
use mongodb::Client;
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
let uri = get_mongo_uri(cfg);
let uri = get_mongo_uri(cfg)?;
let mut options = mongodb::options::ClientOptions::parse(&uri).await?;
options.server_selection_timeout = Some(std::time::Duration::from_secs(3));
options.connect_timeout = Some(std::time::Duration::from_secs(3));
@@ -15,14 +15,31 @@ pub fn select_mongo_path() -> std::path::PathBuf {
"/usr/local/mongodb/bin".to_string().into()
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> String {
if cfg.username.is_empty() {
format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database)
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!(
"mongodb://{}:{}/{}",
cfg.host, cfg.port, cfg.database
))
} else {
format!(
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
)
))
}
}
pub fn extract_db_name(dry_output: &str) -> Option<String> {
let mut dbs = std::collections::HashSet::new();
for line in dry_output.lines() {
if let Some(pos) = line.find("archive prelude ") {
let rest = &line[pos + "archive prelude ".len()..];
if let Some(dot) = rest.find('.') {
let db = &rest[..dot];
dbs.insert(db.to_string());
}
}
}
dbs.into_iter().next()
}
+6 -9
View File
@@ -1,9 +1,11 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -27,21 +29,16 @@ impl Database for MongoDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
)
.await;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod backup;
mod restore;
mod connection;
pub mod database;
mod ping;
mod connection;
mod restore;
+10 -4
View File
@@ -1,14 +1,20 @@
#![allow(dead_code)]
use crate::domain::mongodb::connection::connect;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use mongodb::bson::doc;
use tracing::{error};
use crate::domain::mongodb::connection::connect;
use tracing::error;
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let client = connect(cfg.clone()).await?;
let db_name = if cfg.username.is_empty() { &cfg.database } else { "admin" };
let db_name = if cfg.username.is_empty() && cfg.password.is_empty() {
&cfg.database
} else {
"admin"
};
match client.database(db_name).run_command(doc! {"ping": 1}).await {
Ok(_) => Ok(true),
Err(e) => {
@@ -18,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
error!("Full Error: {}", e);
error!("Check you database network connectivity");
error!("----------------------------------------");
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
Ok(false)
}
}
}
+52 -10
View File
@@ -1,32 +1,74 @@
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
use crate::domain::mongodb::connection::{extract_db_name, get_mongo_uri, select_mongo_path};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting MongoDB restore for database {}", cfg.name);
logger.log("debug", format!("Starting MongoDB restore for database {}", cfg.name));
let mongorestore = select_mongo_path().join("mongorestore");
let uri = get_mongo_uri(cfg.clone());
let uri = get_mongo_uri(cfg.clone())?;
let output = Command::new(mongorestore)
let dry_start = Instant::now();
let dry_run = Command::new(&mongorestore)
.arg(format!(
"--uri={}",
format!(
"mongodb://{}:{}@{}:{}/?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port
)
))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--dryRun")
.arg("--verbose")
.output()?;
let dry_duration_ms = dry_start.elapsed().as_millis() as f64;
let dry_exit_code = dry_run.status.code().unwrap_or(-1);
let dry_output = String::from_utf8_lossy(&dry_run.stderr);
logger.log_command(
"mongorestore --dryRun",
if dry_output.is_empty() { None } else { Some(dry_output.to_string()) },
Some(dry_exit_code),
Some(dry_duration_ms),
);
let source_db = extract_db_name(&dry_output).unwrap_or_else(|| {
logger.log("info", format!("Could not detect source database from archive, falling back to configured database: {}", cfg.database));
cfg.database.clone()
});
logger.log("info", format!("Using source database in archive: {}", source_db));
let start = Instant::now();
let output = Command::new(&mongorestore)
.arg(format!("--uri={}", uri))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--drop")
.arg(format!("--nsInclude={}.*", source_db))
.arg(format!("--nsFrom={}.*", source_db))
.arg(format!("--nsTo={}.*", cfg.database))
.output()
.with_context(|| format!("Failed to run mongorestore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MongoDB restore failed for {}: {}", cfg.name, stderr);
anyhow::bail!("MongoDB restore failed for : {}", cfg.name);
logger.log_command("mongorestore", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MongoDB restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MongoDB restore failed for: {}", cfg.name);
}
info!("MongoDB restore completed for {}", cfg.name);
logger.log_command("mongorestore", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("MongoDB restore completed for {}", cfg.name));
Ok(())
})
.await?
+56
View File
@@ -0,0 +1,56 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting MSSQL backup for database {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let connection_string = format!(
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
);
logger.log("info", format!("MSSQL backup: {}:{}/{}{}", cfg.host, cfg.port, cfg.database, file_path.display()));
let start = Instant::now();
let output = Command::new("sqlpackage")
.arg("/a:Export")
.arg(format!("/scs:{}", connection_string))
.arg(format!("/tf:{}", file_path.display()))
.output()
.with_context(|| format!("Failed to run sqlpackage for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
if !output.status.success() {
logger.log("error", format!("MSSQL backup failed — stderr: {} stdout: {}", stderr, stdout));
let out = format!("stderr: {} stdout: {}", stderr, stdout);
logger.log_command("sqlpackage", Some(out), Some(exit_code), Some(duration_ms));
anyhow::bail!("MSSQL backup failed for {}: {}", cfg.name, stderr);
}
let combined = if stdout.is_empty() && stderr.is_empty() {
None
} else {
Some(format!("{}{}", stdout, stderr).trim().to_string())
};
logger.log_command("sqlpackage", combined, Some(0), Some(duration_ms));
logger.log("info", format!("MSSQL backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+18
View File
@@ -0,0 +1,18 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use tiberius::{AuthMethod, Client, Config};
use tokio::net::TcpStream;
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
pub async fn build_client(cfg: &DatabaseConfig) -> Result<Client<Compat<TcpStream>>> {
let mut config = Config::new();
config.host(&cfg.host);
config.port(cfg.port);
config.authentication(AuthMethod::sql_server(&cfg.username, &cfg.password));
config.trust_cert();
let tcp = TcpStream::connect(config.get_addr()).await?;
tcp.set_nodelay(true)?;
let client = Client::connect(config, tcp.compat_write()).await?;
Ok(client)
}
+50
View File
@@ -0,0 +1,50 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MssqlDatabase {
cfg: DatabaseConfig,
}
impl MssqlDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for MssqlDatabase {
fn file_extension(&self) -> &'static str {
".bacpac"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod database;
mod connection;
mod ping;
mod backup;
mod restore;
+24
View File
@@ -0,0 +1,24 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use tracing::{error, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
info!("Running ping for MSSQL database {}", cfg.name);
match super::connection::build_client(&cfg).await {
Ok(mut client) => match client.simple_query("SELECT 1").await {
Ok(_) => {
info!("MSSQL ping succeeded for {}", cfg.name);
Ok(true)
}
Err(e) => {
error!("MSSQL ping query failed for {}: {:?}", cfg.name, e);
Ok(false)
}
},
Err(e) => {
error!("MSSQL connection failed for {}: {:?}", cfg.name, e);
Ok(false)
}
}
}
+54
View File
@@ -0,0 +1,54 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name));
let connection_string = format!(
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
);
logger.log("info", format!(
"MSSQL restore: {} → {}:{}/{}",
restore_file.display(),
cfg.host,
cfg.port,
cfg.database
));
let start = Instant::now();
let output = Command::new("sqlpackage")
.arg("/a:Import")
.arg(format!("/tcs:{}", connection_string))
.arg(format!("/sf:{}", restore_file.display()))
.output()
.with_context(|| format!("Failed to run sqlpackage restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
if !output.status.success() {
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined.clone()) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!(
"MSSQL restore failed for {} — stderr: {} stdout: {}",
cfg.name, stderr, stdout
));
anyhow::bail!("MSSQL restore failed for {}: {}", cfg.name, stderr);
}
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("MSSQL restore completed for {}", cfg.name));
Ok(())
})
.await?
}
+37 -19
View File
@@ -1,62 +1,80 @@
use crate::domain::mysql::connection::server_version;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
let _version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Mysql version detected: {}", v);
logger.log("debug", format!("MySQL version detected: {}", v));
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
info!("Mysql version found: {}", version);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
if let Ok(out) = Command::new("mysqldump").arg("--version").output() {
logger.log("debug", format!("mysqldump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
}
logger.log("info", format!("Running mysqldump for {}", cfg.name));
let start = Instant::now();
let output = Command::new("mysqldump")
.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--verbose")
.arg("--single-transaction")
.arg("--set-gtid-purged=OFF")
.arg("--no-tablespaces")
.arg("--quick")
.arg("--add-drop-database")
.arg("--databases")
.arg(cfg.database)
.arg("-r")
.arg(&file_path)
.arg("--skip-lock-tables")
.arg("--skip-add-drop-table")
.arg("--no-create-db")
.arg("--default-character-set=utf8mb4")
.arg("--network-timeout")
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
.arg(&cfg.database)
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mysqldump for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let _stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
logger.log_command("mysqldump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
}
logger.log_command("mysqldump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("mysqldump completed for {}", cfg.name));
Ok(file_path)
})
.await?
+10 -6
View File
@@ -1,13 +1,17 @@
use crate::services::config::DatabaseConfig;
use std::process::Command;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mysql")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("-e").arg("SELECT VERSION();")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
@@ -18,7 +22,7 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1) // skip column header
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
+19 -13
View File
@@ -1,14 +1,13 @@
use std::collections::HashMap;
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
ping, restore,
};
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MySQLDatabase {
cfg: DatabaseConfig,
@@ -21,7 +20,7 @@ impl MySQLDatabase {
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.clone());
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
envs
}
}
@@ -36,16 +35,23 @@ impl Database for MySQLDatabase {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod restore;
mod ping;
mod connection;
mod restore;
+15 -8
View File
@@ -1,19 +1,26 @@
use crate::services::config::DatabaseConfig;
use anyhow::Context;
use std::collections::HashMap;
use std::process::Command;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let output = Command::new("mysqladmin")
.arg("--host")
let mut cmd = Command::new("mariadb-admin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.arg("ping")
.envs(env)
.output()
.with_context(|| format!("Failed to ping MySQL server {}", cfg.name))?;
Ok(output.status.success())
.envs(env);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output?;
Ok(output.status.success())
}
Err(_) => Ok(false),
}
}
+34 -24
View File
@@ -1,28 +1,26 @@
use std::fs::File;
use std::io::{Read, Write};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use tracing::{debug, error, info};
use std::fs::File;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
use crate::services::config::DatabaseConfig;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
logger.log("info", format!("Starting restore for database {}", cfg.name));
let mut sql_content = String::new();
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
file.read_to_string(&mut sql_content)
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
cfg.database
);
let drop_status = Command::new("mysql")
let drop_start = Instant::now();
let drop_output = Command::new("mysql")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
@@ -32,14 +30,22 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.status()
.output()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
if !drop_status.success() {
error!("Drop/create database failed for {}", cfg.name);
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
if !drop_output.status.success() {
logger.log_command("mysql", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
info!("Database {} dropped and recreated", cfg.name);
logger.log_command("mysql", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
let start = Instant::now();
let mut child = Command::new("mysql")
.arg("--host")
@@ -48,6 +54,7 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.arg("--database")
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
@@ -55,27 +62,30 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
stdin.write_all(sql_content.as_bytes())
.context("Failed to write SQL content to mysql stdin")?;
stdin.flush()?;
std::io::copy(&mut file, &mut stdin)
.context("Failed to stream SQL content to mysql stdin")?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete mysql restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MySQL restore failed for {}: {}", cfg.name, stderr);
logger.log_command("mysql", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MySQL restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MySQL restore failed for {}", cfg.name);
}
info!("Restore finished successfully for database {}", cfg.name);
logger.log_command("mysql", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
Ok(())
});
handle
.await??;
handle.await??;
Ok(())
}

Some files were not shown because too many files have changed in this diff Show More