Compare commits

...

230 Commits

Author SHA1 Message Date
Charles GTE ff47024d9a fix: mysql localhost 2026-08-21 19:05:47 +02:00
github-actions[bot] 6537e9df53 chore: release 1.19.0 2026-08-20 11:56:59 +00:00
Charles GTE b8d869d5a6 Merge pull request #97 from Portabase/feat/dashboard-databases
feat: dashboard-databases
2026-08-20 13:54:46 +02:00
Charles GTE 90941ea67d fix 2026-08-20 13:41:14 +02:00
charles-gauthereau 046d593e2b fix 2026-08-18 18:20:21 +02:00
charles-gauthereau cf9a59a138 fix: config.rs 2026-08-18 13:07:22 +02:00
charles-gauthereau 2464f6dfb0 Merge branch 'main' into feat/dashboard-databases
# Conflicts:
#	docker-compose.yml
#	src/services/config.rs
2026-08-18 13:05:56 +02:00
github-actions[bot] 069067ca55 chore: release 1.18.6 2026-08-17 09:10:08 +00:00
Charles GTE 04b654d219 Merge pull request #96 from Portabase/fix/mongo-cloud
fix: mongodb cloud cluster issue
2026-08-17 11:07:35 +02:00
charles-gauthereau d83511cb64 fix: mongodb cloud cluster issue 2026-08-17 10:52:21 +02:00
charles-gauthereau ca294e968c fix 2026-08-13 22:02:17 +02:00
charles-gauthereau 1be88ffdb9 Merge branch 'main' into feat/dashboard-databases
# Conflicts:
#	docker-compose.yml
2026-08-13 21:56:38 +02:00
charles-gauthereau 9d393a96a4 fix 2026-08-13 21:55:59 +02:00
charles-gauthereau 044bf80633 feat(agent): ingest dashboard databases via merged cycle + cache 2026-08-13 18:47:18 +02:00
charles-gauthereau 0a6eb6db22 feat(dashboard_config): atomic cache load/persist 2026-08-13 18:40:56 +02:00
charles-gauthereau b26ff81889 feat(dashboard_config): merge (dashboard-wins) + collect_configs 2026-08-13 18:39:54 +02:00
charles-gauthereau 4e2f29f4ca feat(status): decrypt dashboard config_ciphertext into resolved_config 2026-08-13 18:38:15 +02:00
charles-gauthereau d1c8df4cac refactor(config): extract build_config, add load_optional + Serialize 2026-08-13 18:35:52 +02:00
github-actions[bot] 30f83bafcf chore: release 1.18.5 2026-07-26 09:06:42 +00:00
Charles GTE de106c835e Merge pull request #93 from Portabase/fix/s3-storage
fix: s3 logs
2026-07-26 11:04:35 +02:00
Charles GTE 23d6822ddc fix: s3 logs 2026-07-26 11:04:07 +02:00
github-actions[bot] fe1d74945f chore: release 1.18.4 2026-07-25 14:03:12 +00:00
Charles GTE 424a646385 Merge pull request #92 from Portabase/fix/windows-build
fix: windows-build
2026-07-25 16:00:57 +02:00
Charles GTE 0ef4bba5d7 fix: docker-compose.yml 2026-07-25 15:45:56 +02:00
Charles GTE 6548140eaf fix: windows build 2026-07-25 15:19:46 +02:00
github-actions[bot] c9725c381e chore: release 1.18.3 2026-07-24 16:46:51 +00:00
Charles GTE 03695b1897 Merge pull request #91 from Portabase/fix/firebird
fix: firebird
2026-07-24 18:44:20 +02:00
charles-gauthereau 80b1c0dac0 fix: firebird with -g for gbak 2026-07-24 18:29:04 +02:00
charles-gauthereau 5ededb3764 Merge branch 'main' into dev 2026-07-24 17:45:46 +02:00
github-actions[bot] 54cfd673e4 chore: release 1.18.2 2026-07-23 18:04:50 +00:00
Charles GTE 39dbeb5d30 Merge pull request #90 from Portabase/fix/ghcr-ci-cd
fix: ghcr.yml
2026-07-23 20:02:28 +02:00
charles-gauthereau 380cea79fb fix: ghcr.yml 2026-07-23 20:02:01 +02:00
github-actions[bot] 65e4538c2c chore: release 1.18.1 2026-07-23 17:47:38 +00:00
Charles GTE 86a706bca5 Merge pull request #89 from Portabase/fix/ghcr-ci-cd
fix: ghcr.yml
2026-07-23 19:45:23 +02:00
charles-gauthereau e8bda60f16 fix: ghcr.yml 2026-07-23 18:47:41 +02:00
charles-gauthereau 034912b176 Merge branch 'main' into dev 2026-07-23 18:24:18 +02:00
github-actions[bot] 4fd50ccda4 chore: release 1.18.0 2026-07-23 15:33:29 +00:00
Charles GTE 84c6b765c8 Merge pull request #88 from Portabase/fix/postgres-provider
fix: postgres-provider
2026-07-23 17:31:02 +02:00
charles-gauthereau 9790837200 fix: postgres 2026-07-23 17:14:27 +02:00
charles-gauthereau 8827234344 refactor(postgres): split restore into restore/ module (run, prepare, command, toc) 2026-07-23 16:40:56 +02:00
charles-gauthereau b3417304bb fix(postgres): match real pg_restore -l schema line in toc_creates_public_schema 2026-07-23 10:06:01 +02:00
charles-gauthereau 5f4006b1ca feat(postgres): sniff dump format by magic bytes 2026-07-23 09:56:46 +02:00
charles-gauthereau f24b1ec3e4 feat(postgres): wire clean_mode=drop_database with preflight gate 2026-07-23 09:51:49 +02:00
charles-gauthereau bb1b16d0f2 feat(postgres): add drop_and_recreate_database + preflight + FORCE/retry 2026-07-23 09:43:24 +02:00
charles-gauthereau 2aca036ee4 feat(postgres): wire clean_mode none/clean/drop_schemas into restore 2026-07-23 09:36:25 +02:00
charles-gauthereau 2a72a55e57 feat(postgres): detect CREATE SCHEMA public in archive TOC 2026-07-23 09:31:50 +02:00
charles-gauthereau ae63a0df50 feat(postgres): add drop_all_schemas + recreate_public_schema 2026-07-23 09:27:36 +02:00
charles-gauthereau 560c5e02f2 feat(postgres): add RestoreCleanMode option parsing 2026-07-23 09:24:21 +02:00
charles-gauthereau ca17bf9cb9 feat(postgres): add quote_ident/quote_literal for DDL 2026-07-23 09:20:15 +02:00
charles-gauthereau 9fbaab6feb refactor(postgres): unify FC/FD restore via run_pg_restore, validate-before-destroy 2026-07-23 09:15:32 +02:00
charles-gauthereau 1289497d98 refactor(postgres): extract prepare_archive, validate before restore 2026-07-23 09:09:19 +02:00
charles-gauthereau 27c55dbfd8 refactor(postgres): drive async restore work via captured Tokio Handle 2026-07-23 09:01:50 +02:00
charles-gauthereau 45e3acd709 fix(postgres): add pg_restore_binary_name() for Windows restore 2026-07-23 08:58:49 +02:00
charles-gauthereau a93054518a Merge branch 'main' into dev 2026-07-22 00:06:17 +02:00
Charles GTE baf2edc829 fix (#87)
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-21 23:58:11 +02:00
github-actions[bot] f050fa35e0 chore: release 1.17.1 2026-07-21 21:29:30 +00:00
Charles GTE 90d55f7630 fix: backup folder name (#86)
* fix: support configurable backup file prefix (#83)

* fix: backup folder name

* fix: ghcr publish agent image

---------

Co-authored-by: Antonin Jousson <18756890+Antoninj@users.noreply.github.com>
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-21 23:27:10 +02:00
charles-gauthereau c186d57105 fix: ghcr publish agent image 2026-07-21 23:10:40 +02:00
charles-gauthereau 37b4c1fe6d fix: backup folder name 2026-07-21 23:05:59 +02:00
Antonin Jousson 92a3324ca2 fix: support configurable backup file prefix (#83) 2026-07-21 21:50:11 +02:00
github-actions[bot] 82ffb71496 chore: release 1.17.0 2026-07-11 10:08:21 +00:00
Charles GTE 20b3b58024 Merge pull request #80 from Portabase/fix/storage-encryption
fix/storage-encryption
2026-07-11 12:05:59 +02:00
charles-gauthereau 6cddc73565 Merge branch 'main' into fix/storage-encryption
# Conflicts:
#	docker-compose.yml
2026-07-10 18:49:01 +02:00
github-actions[bot] fd3b0076c0 chore: release 1.16.3 2026-07-09 09:11:23 +00:00
Charles GTE 334ab51125 fix: cron (#79)
* fix: cron log

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-09 11:03:55 +02:00
github-actions[bot] 4df3fe2d0c chore: release 1.16.2 2026-07-09 07:38:49 +00:00
Charles GTE ec7715ca26 fix: cron (#78)
* fix: docker-compose.yml

* fix: cron

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-07-09 09:36:53 +02:00
charles-gauthereau 32d4e22196 fix: status.rs 2026-07-08 22:24:50 +02:00
charles-gauthereau f277c5485f feat: decrypt encrypted storages in status ping 2026-07-08 20:52:17 +02:00
charles-gauthereau 585e0bea72 feat: accept encrypted storages markers in DatabaseStatus 2026-07-08 20:47:20 +02:00
charles-gauthereau ee97c6056a feat: add decrypt_json_gcm for encrypted status storages 2026-07-08 20:43:01 +02:00
charles-gauthereau b37a4f2180 fix: docker-compose.yml 2026-07-07 18:43:30 +02:00
github-actions[bot] 1d15f40662 chore: release 1.16.1 2026-07-04 10:26:50 +00:00
Charles GTE 324a2ea3f2 fix: security (#75) 2026-07-04 12:24:37 +02:00
github-actions[bot] 9ec92af6c7 chore: release 1.16.0 2026-07-03 06:16:42 +00:00
Charles GTE 1dafdca2a0 Merge pull request #74 from Portabase/feat/docker-volume-backup-restore
feat: docker-volume-backup-restore
2026-07-03 08:14:30 +02:00
charles-gauthereau d12817a960 fix: codecov.yml 2026-07-02 22:42:02 +02:00
charles-gauthereau 608b82a254 fix: test docker_volume.rs 2026-07-02 22:08:55 +02:00
charles-gauthereau bbecca92f9 fix: databases.json 2026-07-02 21:51:17 +02:00
charles-gauthereau b2b2733c4d feat: docker-volume provider
refactor: move build_tar to utils::compress

refactor: move choose_restore_path to utils::common

docs: remove docker-volume README section

fix: keep archive path for multi-file non-docker-volume restores

docs: trim docker-volume README section to essentials

refactor: run docker-volume backup/restore inside spawn_blocking like other providers

docs: enable docker socket, add volume example and security notes

feat: sweep orphaned ephemeral helper containers on startup

feat: docker-volume clean-replace restore via upload_to_container

fix: serialize env-var access in docker-volume tests to avoid setenv/getenv UB

feat: docker-volume backup via download_from_container

feat: docker-volume ping via inspect_volume

fix: return extraction dir for multi-file restore archives

feat: gzip already-tar inputs directly instead of double-wrapping

feat: docker helper (client, self-image, container lifecycle, sweep)

feat: docker-volume provider
2026-07-02 21:45:11 +02:00
github-actions[bot] 2c257c5a58 chore: release 1.15.0 2026-06-29 17:14:03 +00:00
Charles GTE c24c0d7058 Merge pull request #73 from Portabase/fix/postgres-ownership
fix: postgres-ownership
2026-06-29 19:12:01 +02:00
charles-gauthereau 34f353b68f fix: postgres keep_ownership flag 2026-06-29 19:10:49 +02:00
charles-gauthereau ee51996401 chore: document keep_ownership option in example databases.json 2026-06-29 18:54:01 +02:00
charles-gauthereau 694b463787 feat: conditionally omit --no-owner/--no-privileges in pg_restore based on keep_ownership option 2026-06-29 18:52:37 +02:00
charles-gauthereau aed1e86956 feat: add generic options map to DatabaseConfig
Add `options: HashMap<String, serde_json::Value>` to DatabaseConfig and
`options: Option<HashMap<...>>` to InputDatabaseConfig so callers can
pass arbitrary per-database options (e.g. keep_ownership for pg_dumpall).
Update all DatabaseConfig struct literals in test files and add three
new config-service unit tests covering present, absent, and non-bool values.
2026-06-29 18:49:01 +02:00
charles-gauthereau 6388aff1e3 fix: postgres cluster dump command 2026-06-29 17:10:11 +02:00
github-actions[bot] 39a77b18a4 chore: release 1.14.1 2026-06-29 13:22:10 +00:00
Charles GTE e62167182e Merge pull request #72 from Portabase/fix/postgres-cluster
fix: postgres cluster dump command
2026-06-29 15:19:54 +02:00
charles-gauthereau 4be54a614e fix: postgres cluster dump command 2026-06-29 15:18:52 +02:00
github-actions[bot] c5eaa5086c chore: release 1.14.0 2026-06-27 19:16:46 +00:00
Charles GTE 610d443afc Dev (#69)
* fix: pgdump-cluster (#68)

* feat: add as_str/from_str to PostgresDumpFormat

* feat: resolve pg_dumpall/psql binary names

* feat: add include_globals field to database config

* feat: add pg_dumpall/psql globals dump and apply

* feat: add postgres backup bundle (manifest + build + resolve)

* feat: bundle globals into postgres backup when include_globals is set

* feat: replay globals before pg_restore when backup archive is a bundle

* refactor: bind FD restore tempdir guard once to clear unused warnings

* docs: demonstrate include_globals in sample databases.json

* chore: silence test-only re-export warning in non-test builds

* revert: remove include_globals feature, restore plain pg_dump/pg_restore

* feat: add pg_dumpall/psql binary names and is_superuser check

* feat: add postgresql-cluster db type and config parsing

* feat: pg_dumpall cluster backup and psql restore

* feat: route postgresql-cluster through PostgresClusterDatabase

* docs: add postgresql-cluster sample to databases.json

* refactor: split cluster mode into cluster/ module (backup, restore, database)

* test: mirror cluster tests into src/tests/domain/cluster/

* feat: add-gcs-storage-provider (#70)

* fix: refactoring

* chore: add google-cloud-storage and google-cloud-auth deps

* feat: add GCS provider config model

* feat: add GCS credential, client, and stream-source helpers

StreamSource bridges build_stream's Send-only byte stream into the SDK's
StreamingSource (which send_buffered requires to be Send+Sync+'static) via a
bounded mpsc channel, avoiding any change to the shared UploadStream type.

* feat: implement GCS StorageProvider upload

* feat: register google-cloud-storage provider in factory

* test: GCS upload roundtrip against fake-gcs-server

* fix: format GCS bucket as projects/_/buckets/<name> in upload_with_client

write_object rejects the bare bucket id with "malformed bucket name"; the
production provider passed config.bucket_name unformatted, so real uploads
would always fail. Format once in upload_with_client so prod and the
fake-gcs-server test share the corrected path.

* style: cargo fmt GCS provider files

* fix: gcs

* fix: azure (#71)
2026-06-27 21:14:21 +02:00
Charles GTE 16a37e033e fix: azure (#71) 2026-06-27 20:51:47 +02:00
Charles GTE 29ca5a317b feat: add-gcs-storage-provider (#70)
* fix: refactoring

* chore: add google-cloud-storage and google-cloud-auth deps

* feat: add GCS provider config model

* feat: add GCS credential, client, and stream-source helpers

StreamSource bridges build_stream's Send-only byte stream into the SDK's
StreamingSource (which send_buffered requires to be Send+Sync+'static) via a
bounded mpsc channel, avoiding any change to the shared UploadStream type.

* feat: implement GCS StorageProvider upload

* feat: register google-cloud-storage provider in factory

* test: GCS upload roundtrip against fake-gcs-server

* fix: format GCS bucket as projects/_/buckets/<name> in upload_with_client

write_object rejects the bare bucket id with "malformed bucket name"; the
production provider passed config.bucket_name unformatted, so real uploads
would always fail. Format once in upload_with_client so prod and the
fake-gcs-server test share the corrected path.

* style: cargo fmt GCS provider files

* fix: gcs
2026-06-27 20:37:22 +02:00
Charles GTE 01f5d34c96 fix: pgdump-cluster (#68)
* feat: add as_str/from_str to PostgresDumpFormat

* feat: resolve pg_dumpall/psql binary names

* feat: add include_globals field to database config

* feat: add pg_dumpall/psql globals dump and apply

* feat: add postgres backup bundle (manifest + build + resolve)

* feat: bundle globals into postgres backup when include_globals is set

* feat: replay globals before pg_restore when backup archive is a bundle

* refactor: bind FD restore tempdir guard once to clear unused warnings

* docs: demonstrate include_globals in sample databases.json

* chore: silence test-only re-export warning in non-test builds

* revert: remove include_globals feature, restore plain pg_dump/pg_restore

* feat: add pg_dumpall/psql binary names and is_superuser check

* feat: add postgresql-cluster db type and config parsing

* feat: pg_dumpall cluster backup and psql restore

* feat: route postgresql-cluster through PostgresClusterDatabase

* docs: add postgresql-cluster sample to databases.json

* refactor: split cluster mode into cluster/ module (backup, restore, database)

* test: mirror cluster tests into src/tests/domain/cluster/
2026-06-27 17:23:57 +02:00
github-actions[bot] 0b7b7ef59b chore: release 1.13.1 2026-06-25 12:58:08 +00:00
Charles GTE 038c5523b7 fix: download-crash-large-files (#67)
* fix(restore): add ProgressTracker for download progress logging

* fix(restore): stream backup download to disk instead of buffering in RAM

* chore(restore): add download diagnostics + exact byte logging

* refactor(restore): drop ProgressTracker, keep streaming download

* feat(restore): drive download progress from RestoreInfo.size

* fix(restore): accept integer or string for RestoreInfo.size

* fix(restore): show raw bytes in download progress log

* fix(restore): collapse progress to highest milestone per chunk

* fix: downloader.rs
2026-06-25 14:55:49 +02:00
github-actions[bot] ea010ce713 chore: release 1.13.0 2026-06-21 14:41:22 +00:00
Charles GTE 8897568281 Merge pull request #66 from Portabase/feat/az-blob-storage-2
feat: az-blob-storage
2026-06-21 16:39:12 +02:00
Charles GTE 66cad4e12e fix: az blog storage tests 2026-06-21 16:22:51 +02:00
Charles GTE 221ed4e7e1 feat: add azure blob storage support 2026-06-20 19:38:00 +02:00
Charles GTE 84735ac399 feat(storage): spike Azure Blob SAS auth + block round-trip against Azurite
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-20 17:49:54 +02:00
Charles GTE f7639de096 build: add azure_storage_blob and azure_core dependencies 2026-06-20 17:39:10 +02:00
github-actions[bot] e504d09cb3 chore: release 1.12.3 2026-06-20 13:00:11 +00:00
Charles GTE 0363b300a4 Merge pull request #65 from Portabase/dev
Dev
2026-06-20 14:58:03 +02:00
Charles GTE bf6e7d41ab fix: Dockerfile 2026-06-20 14:40:37 +02:00
Charles GTE 2c7065ae95 Merge pull request #64 from pcrosthwaite/feature-backup-timeout-on-large-db
Support large MySQL and MariaDb databases
2026-06-20 14:38:26 +02:00
Charles GTE 1fece577cc fix: tests and config.rs and backup.rs args 2026-06-20 14:36:09 +02:00
Charles GTE 273475fa3b Merge branch 'dev' into feature-backup-timeout-on-large-db 2026-06-20 13:02:14 +02:00
Charles GTE 628c017584 Merge pull request #63 from tobfel/tf-main
docs: add Windows build and run guide / and fix(postgres): resolve pg_dump/pg_restore path cross-platform
2026-06-20 12:53:31 +02:00
Charles GTE f02218708a fix: windows-release.yml 2026-06-20 12:52:53 +02:00
Charles GTE 16152328b0 fix: extract the env variable to the settings.rs and some refactoring, and the trigger in windows-release.yml 2026-06-20 12:31:01 +02:00
Peter Crosthwaite 7328827435 Fix mysqldump timeout on large database backups
- Add mysql:8.4 client tools stage to Dockerfile so the real mysqldump
  binary (with --network-timeout) takes PATH precedence over MariaDB's
  mysqldump symlink, which lacks this flag
- Use --network-timeout in MySQL backup command; sets net_read_timeout and
  net_write_timeout to 86400s for the duration of the dump
- Simplify config: replace net_read_timeout/net_write_timeout/max_allowed_packet
  with a single max_packet_size field (optional, defaults to 512M)
- Log mysqldump/mariadb-dump client version as a debug message before each backup
2026-06-20 18:06:21 +10:00
Peter Crosthwaite 348eaac81b Added config options for setting timeouts on mysql/mariadb 2026-06-20 07:51:18 +10:00
tf 0f6c93ecd0 fix encoding 2026-06-19 16:31:21 +02:00
tf befec0deae Windows Build workflow 2026-06-19 16:01:45 +02:00
tf 2052ab0ff5 fix(postgres): resolve pg_dump/pg_restore path cross-platform 2026-06-19 15:27:45 +02:00
tf 461e92d67e docs: add Windows build and run guide 2026-06-19 15:22:24 +02:00
github-actions[bot] 23678bf2d6 chore: release 1.12.2 2026-06-19 10:00:23 +00:00
Charles GTE 02105a8171 Merge pull request #62 from Portabase/fix/password-slash
fix: password-slash
2026-06-19 11:58:26 +02:00
charles-gauthereau f94656a39b fix: postgres password slash 2026-06-19 11:29:51 +02:00
Charles GTE b94ff4e987 Merge branch 'main' into dev 2026-06-09 19:18:38 +02:00
github-actions[bot] dc32c442f3 chore: release 1.12.1 2026-06-07 18:06:37 +00:00
Charles GTE 1453851555 Merge pull request #61 from Portabase/fix/cpu-ram-crash
fix: cpu-ram-crash
2026-06-07 20:04:25 +02:00
github-actions[bot] 185a7ee612 chore: release 1.12.0 2026-06-07 15:46:22 +00:00
Charles GTE d8328f60ca Merge pull request #60 from Portabase/feat/backup-restore-job-logs
feat: backup-restore-job-logs
2026-06-07 17:44:26 +02:00
charles-gauthereau 82c7122790 fix: refactoring 2026-06-07 17:32:16 +02:00
Charles GTE fba72f5c26 feat: add job logs for restoration process 2026-06-07 11:16:02 +02:00
Charles GTE b537980c71 fix: test ci 2026-06-06 21:14:38 +02:00
Charles GTE 0261a07fc2 fix: test ci 2026-06-06 20:59:30 +02:00
Charles GTE c840f91680 fix: test ci 2026-06-06 20:20:52 +02:00
Charles GTE 2d52eccbb2 fix: test ci 2026-06-06 19:43:54 +02:00
Charles GTE f9bf481c79 fix: test ci 2026-06-06 19:26:44 +02:00
Charles GTE 3c853ee7ca fix: tests and refactoring 2026-06-06 19:15:22 +02:00
Charles GTE 45d791358c fix: tracing logs 2026-06-06 11:01:55 +02:00
Charles GTE 0b915d87c9 feat: job logs 2026-06-06 10:22:27 +02:00
Charles GTE 531a25f292 Merge branch 'main' into dev 2026-05-24 16:52:40 +02:00
Charles GTE 41e2b197cb Merge remote-tracking branch 'origin/main' 2026-05-24 16:52:35 +02:00
github-actions[bot] f149363b12 chore: release 1.11.1 2026-05-17 16:51:27 +00:00
Charles GTE aa3ff39bb3 fix: Dockerfile (#58)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

* fix: tests and refactoring, add justfile instead of makefile

* fix: databases.json

* fix: add gbak in production Dockerfile

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-05-17 18:49:18 +02:00
Charles GTE 740ee43038 fix: add gbak in production Dockerfile 2026-05-17 18:26:29 +02:00
Charles GTE 1fccc0bc19 fix: databases.json 2026-05-15 22:17:02 +02:00
Charles GTE 2c15ea64ae Merge branch 'main' into dev 2026-05-15 22:09:41 +02:00
Charles GTE 18bfdf3b7e Merge remote-tracking branch 'origin/main' 2026-05-11 22:09:18 +02:00
github-actions[bot] db26910679 chore: release 1.11.0 2026-05-11 20:07:01 +00:00
Charles GTE eac2853b83 feat: add-mssql-support (#56)
* fix: tests and refactoring, add justfile instead of makefile

* chore: add tiberius dependency for MSSQL support

* feat: add Mssql variant to DbType enum and config validation

* feat: add MSSQL domain adapter with tiberius ping and sqlpackage backup/restore

* feat: install sqlpackage in Docker base and prod stages

* feat: add MSSQL service to docker-compose and config examples

* fix: log backup errors instead of silently discarding them

* fix: use dotnet-install.sh instead of Microsoft apt feed (SHA1 key rejected by Debian Trixie)

* fix: add /TrustServerCertificate:true to sqlpackage commands for self-signed certs

* fix: use connection string with TrustServerCertificate for sqlpackage

* fix: use user database in MSSQL config example (system databases cannot be exported)

* feat: add seed-mssql task and seed script

* fix: docker-compose.yml
2026-05-11 22:04:36 +02:00
Charles GTE 8690a15888 Merge remote-tracking branch 'origin/main' 2026-05-11 19:45:48 +02:00
github-actions[bot] a4bbb37a4e chore: release 1.10.1 2026-04-29 20:42:31 +00:00
Charles GTE ac8f7fd8d8 Merge branch 'main' into dev 2026-04-29 22:41:16 +02:00
Charles GTE c6baa8fdc3 Merge remote-tracking branch 'origin/main'
# Conflicts:
#	docker-compose.yml
2026-04-29 22:40:31 +02:00
Charles GTE 14068846b9 fix: migration-process (#55)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

* fix: tests and refactoring, add justfile instead of makefile

* fix: restoration process for postgres/mysql and mariadb

* fix: restoration process for postgres/mysql and mariadb

* fix: postgres restore

* fix: mariadb/mysql restore, added backticks to drop command

* fix: mongodb and migration test for firebird

* fix: docker-compose.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-29 22:40:02 +02:00
Charles GTE 6edf2890f1 Merge remote-tracking branch 'origin/dev' into dev 2026-04-26 17:42:52 +02:00
Charles GTE 5f579690ff Merge branch 'main' into dev 2026-04-26 17:42:46 +02:00
Charles GTE bfe63d7fd3 fix: tests and refactoring, add justfile instead of makefile 2026-04-26 17:42:30 +02:00
github-actions[bot] 0a422448b4 chore: release 1.10.0 2026-04-16 09:22:34 +00:00
Charles GTE 14a6c9ab0a feat: add chunk size in .env variables (#54)
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-16 11:20:14 +02:00
charles-gauthereau 5c35375df7 Merge branch 'main' into dev 2026-04-08 19:57:08 +02:00
Charles GTE 86027fd6bd fix: docker compose files [skip-release] (#53)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* chore: docker-compose.databases.yml

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-02 19:30:07 +02:00
charles-gauthereau 61c7224104 chore: docker-compose.databases.yml 2026-04-02 19:28:20 +02:00
charles-gauthereau d7958e03b9 Merge branch 'main' into dev 2026-04-01 17:25:32 +02:00
Charles GTE 8eab97a3d6 chore: README.md (#52) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-04-01 17:24:48 +02:00
github-actions[bot] 25c95108f0 chore: release 1.9.0 2026-03-31 13:12:45 +00:00
Charles GTE f37e4d2de9 feat: add firebird (#51)
* feat: add firebird

* feat: add firebird tests

* fix: tests and refactoring, add justfile instead of makefile
2026-03-31 15:10:05 +02:00
Charles GTE b269d98d3c Merge branch 'main' into dev 2026-03-29 09:32:25 +02:00
github-actions[bot] a1aa66ff3f chore: release 1.8.0 2026-03-29 07:30:57 +00:00
Charles GTE d4abcbcebc feat: separate mysql and mariadb (#49)
* feat: separate mysql and mariadb

* feat: separate mysql and mariadb
2026-03-29 09:28:30 +02:00
Charles GTE 2d44b62844 Merge branch 'main' into dev 2026-03-28 17:23:31 +01:00
github-actions[bot] 8bf6503f1f chore: release 1.7.0 2026-03-28 15:47:43 +00:00
Charles GTE fdf35228d4 feat: add-heathcheck-database (#48)
feat: add-heathcheck-database
2026-03-28 16:45:13 +01:00
Charles GTE 1855c5120a fix: docker-compose.yml 2026-03-28 16:44:07 +01:00
Charles GTE 3d5a7eeded feat: add health check ping for databases 2026-03-28 16:07:37 +01:00
Charles GTE 517f3779cd feat: sending database ping result for each databases to portabase server 2026-03-27 23:07:06 +01:00
github-actions[bot] a1e18754b1 chore: release 1.6.3 2026-03-27 21:27:23 +00:00
Charles GTE 092f760431 Merge branch 'main' into dev 2026-03-27 22:25:48 +01:00
Charles GTE 725312e1d5 fix: tests (#47)
* fix: s3 port type mismatch, add string_or_number_to_string deserializer

* fix: adding some tests

* fix: adding some tests

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

* fix: working on pipeline

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-27 22:24:59 +01:00
Charles GTE 2c5c805308 Merge branch 'main' into dev 2026-03-21 10:20:08 +01:00
github-actions[bot] 8e8ebb5920 chore: release 1.6.2 2026-03-21 07:53:35 +00:00
Charles GTE 8489a6f6e3 fix: adding retry config for s3 provider (#46) 2026-03-21 08:51:15 +01:00
Charles GTE 38a3274c43 Merge branch 'main' into dev 2026-03-21 08:34:41 +01:00
github-actions[bot] 3f5b9c91cc chore: release 1.6.1 2026-03-18 18:42:23 +00:00
Charles GTE 1c52f298d2 fix: Enable Link-Time Optimization (LTO) and codegen-units = 1 for Release builds (#45) 2026-03-18 19:39:49 +01:00
Charles GTE bb45c1961f Merge branch 'main' into dev 2026-03-18 19:11:42 +01:00
github-actions[bot] e602c3540f chore: release 1.6.0 2026-03-16 13:38:02 +00:00
charles-gauthereau ee7016d1fa Merge branch 'main' into dev
# Conflicts:
#	docker-compose.yml
2026-03-16 14:36:05 +01:00
Charles GTE b5e8ccac1f feat: add-valkey #43
feat: add-valkey
2026-03-16 14:35:31 +01:00
charles-gauthereau f14a6dc318 fix: docker-compose.yml 2026-03-16 14:32:33 +01:00
charles-gauthereau e1bc77e481 fix: valkey.rs 2026-03-16 14:31:11 +01:00
charles-gauthereau cbc7517603 fix: README.md with Valkey content 2026-03-16 14:28:24 +01:00
charles-gauthereau fc4e4e3881 fix: log in backup.rs 2026-03-16 12:32:57 +01:00
charles-gauthereau 5ec7bc9f28 fix: docker-compose.yml volume for valkey 2026-03-16 12:32:12 +01:00
charles-gauthereau 45c33dd031 feat: add valkey test sequences 2026-03-16 12:30:22 +01:00
charles-gauthereau cddb9d78fa fix: entrypoint.sh 2026-03-16 12:16:03 +01:00
charles-gauthereau cf5de3115c feat: add valkey database engine for backup 2026-03-16 12:11:01 +01:00
charles-gauthereau d6d2ef18f8 fix: internal redis port 2026-03-16 11:24:30 +01:00
github-actions[bot] 987c630c73 chore: release 1.5.1 2026-03-16 07:59:25 +00:00
Charles GTE e9147d6c60 fix: s3 port type mismatch, add string_or_number_to_string deserializer (#42)
Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-16 08:57:00 +01:00
charlesgauthereau c46c301371 fix: s3 port type mismatch, add string_or_number_to_string deserializer 2026-03-16 08:55:16 +01:00
github-actions[bot] 174ead72a3 chore: release 1.5.0 2026-03-15 11:06:29 +00:00
Charles GTE f2a67b3490 feat: add-redis #39
feat: add-redis
2026-03-15 12:04:05 +01:00
charlesgauthereau d4afefbc1e fix: requirements.sh for tests tools 2026-03-14 20:08:26 +01:00
charlesgauthereau be6e147df2 fix: requirements.sh for tests tools 2026-03-14 19:54:15 +01:00
charlesgauthereau 91b6658852 fix: requirements.sh for tests tools 2026-03-14 19:40:01 +01:00
charlesgauthereau f7dcb6b2bb fix: requirements.sh for tests tools 2026-03-14 18:26:26 +01:00
charlesgauthereau ab14f0dcf0 fix: requirements.sh for tests tools 2026-03-14 18:17:43 +01:00
charlesgauthereau 58a0b92710 fix: pipeline codecov.yml 2026-03-14 18:05:29 +01:00
charlesgauthereau 2f24291c0e fix: pipeline codecov.yml 2026-03-14 18:02:06 +01:00
charlesgauthereau 1fdea31430 fix: postgres and redis tests 2026-03-14 17:59:44 +01:00
charlesgauthereau d1d633fd00 feat: add redis database support for backup. 2026-03-13 22:44:46 +01:00
Charles GTE b1c14f098b fix: codecov.yml (#38)
* feat: add codecov.yml

* fix: add codecov.yml

* fix: codecov.yml

---------

Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-13 16:58:51 +01:00
github-actions[bot] 3abb955e24 chore: release 1.4.1 2026-03-13 15:19:42 +00:00
Charles GTE 6d22be9b50 fix: add codecov.yml (#37) [skip-release]
Co-authored-by: charles-gauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-13 16:17:09 +01:00
Charles GTE c2df4ba71e fix: readme badge [skip-release] #36
fix: readme badge  [skip-release]
2026-03-13 15:57:16 +01:00
charles-gauthereau 46b3f2466a fix: readme badge 2026-03-13 13:50:30 +01:00
github-actions[bot] a2be03751f chore: release 1.4.0 2026-03-12 18:23:42 +00:00
Charles GTE 7df9df1605 feat : test coverage for utils functions #35
feat : test coverage for utils functions
2026-03-12 19:21:09 +01:00
charlesgauthereau df5b16a153 fix: docker-compose.yml 2026-03-12 19:16:12 +01:00
charlesgauthereau 824a5d52a3 feat: add test for util functions and start working on database providers tests procedures 2026-03-12 18:56:07 +01:00
charlesgauthereau 1496208db4 Merge branch 'main' into dev 2026-03-09 16:46:00 +01:00
charlesgauthereau 454e1b442f Merge remote-tracking branch 'origin/main' 2026-03-09 16:45:52 +01:00
github-actions[bot] 54bca7452a chore: release 1.3.0 2026-03-09 15:43:44 +00:00
Charles GTE a96cef6cb5 feat: helm chart (#34)
* fix: docker.yml

* feat: add helm chart config

* fix: helm config

* feat: add .github workflow for helm chart

---------

Co-authored-by: charlesgauthereau <charles.gauthereau@soluce-technologies.com>
2026-03-09 16:41:14 +01:00
charlesgauthereau fd9f183166 Merge branch 'main' into dev 2026-03-08 22:02:36 +01:00
charlesgauthereau 94fb2535ec Merge remote-tracking branch 'origin/main' 2026-03-08 21:59:02 +01:00
charlesgauthereau 7fabc9c98c Merge remote-tracking branch 'origin/main' 2026-03-08 21:47:15 +01:00
charlesgauthereau 063b2e1c2c Merge remote-tracking branch 'origin/main' 2026-03-08 21:12:09 +01:00
charlesgauthereau 924177a4fe Merge remote-tracking branch 'origin/main' 2026-03-08 20:52:36 +01:00
charlesgauthereau 678df8e2bc fix: docker.yml 2026-03-08 20:46:47 +01:00
charlesgauthereau 7e0c7c0688 Merge branch 'main' into dev
# Conflicts:
#	docker-compose.yml
2026-03-08 19:48:00 +01:00
charlesgauthereau b9b9fcf6ee Merge branch 'main' into dev 2026-03-08 10:02:34 +01:00
charlesgauthereau 0224605c59 fix: docker-compose.yml 2026-03-08 10:02:17 +01:00
221 changed files with 11989 additions and 1861 deletions
+1 -1
View File
@@ -60,7 +60,7 @@ representative at an online or offline event.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
contact@soluce-technologies.com.
contact@portabase.io.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
+1 -1
View File
@@ -18,7 +18,7 @@ We take security seriously and aim to support the following versions of the proj
If you discover a security vulnerability in this project, we appreciate your help in disclosing it responsibly.
1. **Contact Us**
Please report the vulnerability by emailing **[contact@soluce-technologies.com](mailto:contact@soluce-technologies.com)**. Include the following details:
Please report the vulnerability by emailing **[contact@portabase.io](mailto:contact@portabase.io)**. Include the following details:
- A detailed description of the issue.
- Steps to reproduce the vulnerability (if applicable).
- Any potential impacts or risks.
+90
View File
@@ -0,0 +1,90 @@
name: Codecov Rust
on:
push:
branches: ["main"]
pull_request:
branches: ["main"]
env:
CARGO_TERM_COLOR: always
CARGO_TARGET_DIR: /app/target
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-C instrument-coverage -C link-dead-code"
LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw"
jobs:
coverage:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build test image
run: docker compose -f docker-compose.test.yml build agent-test
- name: Start agent-test container
run: docker compose -f docker-compose.test.yml up -d agent-test
- name: Run tests inside container
run: |
docker compose -f docker-compose.test.yml exec -T \
-e CARGO_TARGET_DIR \
-e CARGO_INCREMENTAL \
-e RUSTFLAGS \
-e LLVM_PROFILE_FILE \
agent-test bash -c "
mkdir -p /app/coverage &&
rm -rf /app/target/* /app/coverage/* &&
cargo test --verbose -- --test-threads=2 &&
sync
"
- name: Verify profraw files exist
run: |
docker compose -f docker-compose.test.yml exec -T \
agent-test bash -c '
count=$(find /app/coverage -type f -name "*.profraw" | wc -l)
echo "profraw files: $count"
test "$count" -gt 0
'
- name: Generate coverage report inside container
run: |
docker compose -f docker-compose.test.yml exec -T \
agent-test bash -c "
rustup component add llvm-tools-preview &&
grcov /app/coverage \
--binary-path /app/target/debug \
-s /app \
--llvm \
-t lcov \
--branch \
--ignore-not-existing \
--ignore '/app/target/*' \
-o /app/lcov.info
"
- name: Copy lcov.info from container to host
run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info
- name: Show basic coverage report info
run: |
echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}')
head -n 30 ./lcov.info || true
test -s ./lcov.info
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
files: ./lcov.info
flags: unittests
name: rust-unit-coverage
verbose: true
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
use_pypi: true
- name: Stop and remove container
if: always()
run: docker compose -f docker-compose.test.yml down --volumes
+134
View File
@@ -0,0 +1,134 @@
name: GHCR Publish
on:
workflow_call:
inputs:
version:
required: true
type: string
ref:
required: true
type: string
add_latest:
required: false
type: boolean
default: false
dockerfile:
required: false
type: string
default: "./docker/Dockerfile"
target:
required: false
type: string
default: "prod"
permissions:
contents: read
packages: write
jobs:
build:
name: Build ${{ matrix.platform }}
runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
strategy:
fail-fast: false
matrix:
platform: [ linux/amd64, linux/arm64 ]
steps:
- name: Prepare vars
id: prep
run: |
ARCH="${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}"
echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
- name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
fetch-depth: 1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push by digest
id: build
uses: docker/build-push-action@v6
with:
context: .
file: ${{ inputs.dockerfile }}
platforms: ${{ matrix.platform }}
target: ${{ inputs.target }}
provenance: false
outputs: type=image,name=${{ steps.prep.outputs.image }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=ghcr-${{ steps.prep.outputs.arch }}
cache-to: type=gha,mode=max,scope=ghcr-${{ steps.prep.outputs.arch }},ignore-error=true
- name: Export digest
env:
DIGEST: ${{ steps.build.outputs.digest }}
run: |
mkdir -p /tmp/digests
touch "/tmp/digests/${DIGEST#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digest-${{ steps.prep.outputs.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge:
name: Create multi-arch manifest
runs-on: ubuntu-latest
needs: build
steps:
- name: Prepare vars
id: prep
run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/agent" >> "$GITHUB_OUTPUT"
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digest-*
merge-multiple: true
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ steps.prep.outputs.image }}
tags: |
type=semver,pattern={{version}},value=${{ inputs.version }}
type=semver,pattern={{major}}.{{minor}},value=${{ inputs.version }}
type=semver,pattern={{major}},value=${{ inputs.version }}
type=raw,value=latest,enable=${{ inputs.add_latest }}
- name: Create and push manifest list
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ steps.prep.outputs.image }}@sha256:%s ' *)
- name: Inspect
run: docker buildx imagetools inspect ${{ steps.prep.outputs.image }}:${{ inputs.version }}
+37
View File
@@ -0,0 +1,37 @@
name: Publish Helm Chart
on:
workflow_call:
inputs:
version:
required: true
type: string
secrets:
GH_TOKEN:
required: true
jobs:
publish-helm:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v4
- name: Package Helm chart
run: |
mkdir -p ./helm-packages
helm package helm \
--version ${{ inputs.version }} \
--app-version ${{ inputs.version }} \
--destination ./helm-packages
- name: Authenticate to GitHub Packages
run: |
echo "${{ secrets.GH_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin
- name: Push Helm chart to GitHub Packages (OCI)
run: |
helm push ./helm-packages/portabase-agent-${{ inputs.version }}.tgz oci://ghcr.io/portabase/charts
+32
View File
@@ -100,11 +100,42 @@ jobs:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
publish-docker-ghcr:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/ghcr.yml
with:
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
add_latest: true
publish-helm:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/helm.yml
with:
version: ${{ needs.create-release.outputs.version }}
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build-windows:
needs: create-release
if: ${{ needs.create-release.result == 'success' }}
uses: ./.github/workflows/windows-release.yml
with:
version: ${{ needs.create-release.outputs.version }}
ref: ${{ needs.create-release.outputs.version }}
draft_tag: ${{ needs.create-release.outputs.draft_tag }}
secrets:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
finalize-release:
needs:
- create-release
- publish-docker
- publish-docker-ghcr
- publish-helm
- build-windows
runs-on: ubuntu-latest
outputs:
release_tag: ${{ steps.publish_release_step.outputs.release_tag }}
@@ -126,6 +157,7 @@ jobs:
notify-discord:
needs:
- publish-docker
- publish-helm
- create-release
- finalize-release
uses: ./.github/workflows/discord.yml
+100
View File
@@ -0,0 +1,100 @@
name: Build Windows release
on:
workflow_call:
inputs:
version:
description: 'Release version (git tag), e.g. 1.18.4'
type: string
required: false
ref:
description: 'Git ref to check out and build'
type: string
required: false
draft_tag:
description: 'Draft GitHub release tag to attach the asset to (e.g. untagged-xxxx). Empty = skip upload.'
type: string
required: false
secrets:
GH_TOKEN:
required: false
workflow_dispatch:
inputs:
ref:
description: 'Git ref to check out and build'
type: string
required: false
jobs:
build-windows:
runs-on: windows-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.ref || github.ref }}
- name: Set up Rust toolchain (MSVC)
uses: dtolnay/rust-toolchain@stable
with:
targets: x86_64-pc-windows-msvc
- name: Cache cargo build
uses: Swatinem/rust-cache@v2
- name: Cache vcpkg installed packages
uses: actions/cache@v4
with:
path: C:\vcpkg\installed
key: vcpkg-openssl-x64-windows-v1
- name: Install OpenSSL (x64) via vcpkg
shell: pwsh
run: |
# windows-latest ships vcpkg preinstalled; the install is a no-op when the
# package is restored from cache.
& "$env:VCPKG_INSTALLATION_ROOT\vcpkg.exe" install openssl:x64-windows
'VCPKG_ROOT=C:\vcpkg' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
'OPENSSL_DIR=C:\vcpkg\installed\x64-windows' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Build (cargo release)
shell: pwsh
run: cargo build --release --bin app
- name: Prepare artifact zip
id: prepare_artifact
shell: pwsh
env:
RELEASE_VERSION: ${{ inputs.version }}
run: |
$tag = $env:RELEASE_VERSION
if (-not $tag) { $tag = $env:GITHUB_SHA }
$exe = "target\release\app.exe"
if (-not (Test-Path $exe)) { Write-Error "Built binary $exe not found in target/release"; exit 1 }
$outDir = "artifact"
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
# Ship under the package name, not the internal bin name "app".
Copy-Item -Path $exe -Destination "$outDir\portabase-agent.exe"
$zipName = "windows-release-$tag.zip"
if (Test-Path $zipName) { Remove-Item $zipName }
Compress-Archive -Path "$outDir\*" -DestinationPath $zipName -Force
"zip=$zipName" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
- name: Upload build artifact
uses: actions/upload-artifact@v4
with:
name: windows-release
path: ${{ steps.prepare_artifact.outputs.zip }}
- name: Attach asset to draft release
if: ${{ inputs.draft_tag != '' }}
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
run: |
gh release upload "${{ inputs.draft_tag }}" "${{ steps.prepare_artifact.outputs.zip }}" --clobber
+5 -1
View File
@@ -3,4 +3,8 @@
/src/data/
.DS_Store
.env
.env
.claude
/docs
+1 -1
View File
@@ -27,5 +27,5 @@ keywords:
- self-hosted
- portabase
license: Apache-2.0
version: 1.2.5
version: 1.19.0
date-released: '2026-02-24'
Generated
+1654 -769
View File
File diff suppressed because it is too large Load Diff
+25 -4
View File
@@ -1,6 +1,6 @@
[package]
name = "portabase-agent"
version = "1.2.5"
version = "1.19.0"
edition = "2024"
[dependencies]
@@ -30,26 +30,47 @@ tokio-postgres = "0.7.15"
futures = "0.3.31"
tracing-appender = "0.2.4"
time = { version = "0.3.44", features = ["macros"] }
mongodb = "3.5.0"
mongodb = "3.5.1"
rand = "0.9.2"
bytes = "1.11.0"
async-stream = "0.3.6"
uuid = { version = "1.20.0", features = ["v4"] }
tokio-util = "0.7.18"
tokio-util = { version = "0.7.18", features = ["compat", "io"] }
tiberius = { version = "0.12", default-features = false, features = ["rustls", "chrono"] }
aws-config = "1.8.13"
aws-sdk-s3 = { version = "1.122.0", features = ["behavior-version-latest"] }
azure_core = "1.0.0"
azure_storage_blob = "1.0.0"
google-cloud-storage = "1.15"
google-cloud-auth = "1.13"
async-compression = { version = "0.4.37", features = ["tokio", "gzip"] }
tokio-tar = "0.3.1"
oauth2 = "5.0.0"
hyper = "1.8.1"
async-http-client = "0.2.0"
aes-gcm = "0.11.0-rc.3"
generic-array = "0.14.7"
futures-util = "0.3.31"
tokio-stream = "0.1.18"
aes = "0.9.0-rc.4"
typenum = "1.19.0"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] }
postgres = "0.19.12"
url = "2.5.8"
percent-encoding = "2.3.2"
bollard = "0.20.0"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }
mockall = "0.13"
testcontainers = "0.27.1"
testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] }
wiremock = "0.6"
[profile.release]
lto = true
codegen-units = 1
strip = "debuginfo"
[[bin]]
name = "app"
-50
View File
@@ -1,50 +0,0 @@
include .env
export $(shell sed 's/=.*//' .env)
.PHONY: seed-mongo seed-mysql seed-postgres
seed-mongo:
@echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
@echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
@echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$$MYSQL_PORT" -u "$$MYSQL_USER" -p"$$MYSQL_PASSWORD" "$$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed.sql
seed-postgres-1gb:
@echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD=$$PG_PASSWORD $$PG_CONTAINER \
psql -U $$PG_USER -d $$PG_DB < ./scripts/postgres/seed-1gb.sql
SQLITE_SEED_FILE := $(if $(filter big,$(SEED)),./scripts/sqlite/seed-big.sql,./scripts/sqlite/seed.sql)
seed-sqlite:
@echo "Seeding Sqlite..."
@echo "Run as root to fix permissions inside the volume"
docker exec -u 0 -it db-sqlite sh -c "chmod -R 777 /workspace/data"
@echo "Create the database file (if it doesnt exist)"
docker exec -u 0 -it db-sqlite sh -c "touch /workspace/data/app.db"
@echo "Seed the database"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < $(SQLITE_SEED_FILE)
@echo "Verify"
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
@echo "Done"
seed-all: seed-mongo seed-mysql seed-postgres seed-postgres-1gb
+7 -2
View File
@@ -13,15 +13,20 @@
[![License: Apache](https://img.shields.io/badge/License-apache-yellow.svg)](LICENSE)
[![Docker Pulls](https://img.shields.io/docker/pulls/portabase/agent?color=brightgreen)](https://hub.docker.com/r/portabase/agent)
[![Helm Chart](https://img.shields.io/badge/Helm-Kubernetes-326ce5?logo=helm&logoColor=white)](https://github.com/Portabase/agent/pkgs/container/charts%2Fportabase-agent)
[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey)](https://github.com/Portabase/portabase)
[![Support Portabase](https://img.shields.io/badge/Support-Portabase-orange)](https://www.buymeacoffee.com/portabase)
[![PostgreSQL](https://img.shields.io/badge/python-3.13-blue.svg)](https://www.python.org/downloads/release/python-3120/)
[![PostgreSQL](https://img.shields.io/badge/PostgreSQL-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/)
[![MySQL](https://img.shields.io/badge/MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/)
[![SQLite](https://img.shields.io/badge/-SQLite-blue?logo=sqlite&logoColor=white)](https://sqlite.org/)
[![Redis](https://img.shields.io/badge/Redis-DC382D?style=flat&logo=Redis&logoColor=white)](https://redis.io/)
[![MariaDB](https://img.shields.io/badge/MariaDB-003545?logo=mariadb&logoColor=white)](https://mariadb.org/)
[![MongoDB](https://img.shields.io/badge/-MongoDB-13aa52?logo=mongodb&logoColor=white)](https://www.mongodb.com/)
[![Valkey](https://img.shields.io/badge/Valkey-6284fc?style=flat&logo=Valkey&logoColor=white)](https://valkey.io/)
[![Firebird](https://img.shields.io/badge/Firebird-f55b14?style=flat&logo=Firebird&logoColor=white)](https://firebirdsql.org/)
[![Self Hosted](https://img.shields.io/badge/self--hosted-yes-brightgreen)](https://github.com/Portabase/portabase)
[![Open Source](https://img.shields.io/badge/open%20source-❤️-red)](https://github.com/Portabase/portabase)
@@ -49,7 +54,7 @@ You have 4 ways to install Portabase Agent:
- Automated CLI (recommended) - [details](https://portabase.io/docs/agent/setup#cli)
- Docker Compose setup - [details](https://portabase.io/docs/agent/setup#docker)
- Kubernetes with Helm (soon)
- Kubernetes with Helm [details](https://portabase.io/docs/agent/setup#helm)
- Development setup - [details](https://portabase.io/docs/agent/setup#development)
**Ensure Docker is installed on your machine before getting started.**
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
View File
+87 -5
View File
@@ -10,10 +10,23 @@
"host": "db-postgres",
"generated_id": "16678159-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 1 - PostgreSQL - BIS",
"database": "devdb2",
"type": "postgresql",
"username": "devuser2",
"password": "changeme2",
"port": 5432,
"host": "db-postgres-2",
"generated_id": "16678159-ff7e-5697-8c83-0adeff214681",
"options": {
"keep_ownership": true
}
},
{
"name": "Test database 2 - MariaDB",
"database": "mariadb",
"type": "mysql",
"type": "mariadb",
"username": "mariadb",
"password": "changeme",
"port": 3306,
@@ -21,7 +34,7 @@
"generated_id": "16678124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 4 - MongoDB Auth",
"name": "Test database 3 - MongoDB Auth",
"database": "testdbauth",
"type": "mongodb",
"username": "root",
@@ -31,7 +44,7 @@
"generated_id": "16678145-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 5 - MongoDB",
"name": "Test database 4 - MongoDB",
"database": "testdb",
"type": "mongodb",
"port": 27017,
@@ -39,16 +52,85 @@
"generated_id": "16678147-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 6 - SQLite DB",
"name": "Test database 5 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data/workspace/data/app.db",
"generated_id": "16678178-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 7 - SQLite DB",
"name": "Test database 6 - SQLite DB",
"type": "sqlite",
"path": "/sqlite-data-2/workspace/data/app.db",
"generated_id": "16678179-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 7 - Redis",
"type": "redis",
"port": 6379,
"host": "db-redis",
"generated_id": "16678166-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 8 - Redis Auth",
"type": "redis",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-redis-auth",
"generated_id": "16678160-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 9 - Valkey",
"type": "valkey",
"port": 6379,
"host": "db-valkey",
"generated_id": "16678560-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 10 - Valkey Auth",
"type": "valkey",
"password": "supersecurepassword",
"port": 6379,
"username": "default",
"host": "db-valkey-auth",
"generated_id": "16678561-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 11 - Mysql",
"database": "mysqldb",
"type": "mysql",
"username": "mysqldb",
"password": "changeme",
"port": 3306,
"host": "db-mysql",
"generated_id": "16656124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 12 - Firebird",
"database": "/var/lib/firebird/data/mirror.fdb",
"type": "firebird",
"username": "alice",
"password": "fake_password",
"port": 3050,
"host": "db-firebird",
"generated_id": "16706124-ff7e-4c97-8c83-0adeff214681"
},
{
"name": "Test database 13 - MsSQL",
"database": "myappdb",
"type": "mssql",
"username": "sa",
"password": "Portabase!Strong1",
"port": 1433,
"host": "db-mssql",
"generated_id": "16706125-ff7e-4c97-8c83-0adeff214682"
},
{
"name": "Test database 14 - Docker Volume",
"type": "docker-volume",
"volume_name": "databases_sqlite-data",
"generated_id": "16706126-ff7e-4c97-8c83-0adeff214690",
"container_name": "db-sqlite"
}
]
}
+77 -3
View File
@@ -29,11 +29,85 @@ host = "db-mongodb-auth"
generated_id = "16678145-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 3 - MongoDB"
name = "Test database 4 - MongoDB"
database = "testdb"
type = "mongodb"
username = ""
password = ""
port = 27017
host = "db-mongodb"
generated_id = "16678147-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 5 - SQLite DB"
type = "sqlite"
path = "/sqlite-data/workspace/data/app.db"
generated_id = "16678178-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 6 - SQLite DB 2"
type = "sqlite"
path = "/sqlite-data-2/workspace/data/app.db"
generated_id = "16678179-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 7 - Redis"
type = "redis"
port = 6379
host = "db-redis"
generated_id = "16678166-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 8 - Redis Auth"
type = "redis"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-redis-auth"
generated_id = "16678160-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 9 - Valkey"
type = "valkey"
port = 6379
host = "db-valkey"
generated_id = "16678560-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 10 - Valkey Auth"
type = "valkey"
password = "supersecurepassword"
username = "default"
port = 6379
host = "db-valkey-auth"
generated_id = "16678561-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 11 - Mysql"
database = "mysqldb"
type = "mysql"
password = "changeme"
username = "mysqldb"
port = 3306
host = "db-mysql"
generated_id = "16656124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 12 - Firebird"
database = "/var/lib/firebird/data/mirror.fdb"
type = "firebird"
password = "fake_password"
username = "alice"
port = 3050
host = "db-firebird"
generated_id = "16706124-ff7e-4c97-8c83-0adeff214681"
[[databases]]
name = "Test database 13 - MSSQL"
database = "myappdb"
type = "mssql"
username = "sa"
password = "Portabase!Strong1"
port = 1433
host = "db-mssql"
generated_id = "16706125-ff7e-4c97-8c83-0adeff214682"
+213
View File
@@ -0,0 +1,213 @@
name: databases
services:
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
db-postgres-2:
container_name: db-postgres-2
image: postgres:17-alpine
ports:
- "5438:5432"
volumes:
- postgres-data-2:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb2
- POSTGRES_USER=devuser2
- POSTGRES_PASSWORD=changeme2
networks:
- portabase
db-mariadb:
container_name: db-mariadb
image: mariadb:latest
ports:
- "3311:3306"
environment:
- MYSQL_DATABASE=mariadb
- MYSQL_USER=mariadb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mariadb-data:/var/lib/mysql
networks:
- portabase
db-mysql:
container_name: db-mysql
image: mysql:9.5
ports:
- "3312:3306"
environment:
- MYSQL_DATABASE=mysqldb
- MYSQL_USER=mysqldb
- MYSQL_PASSWORD=changeme
- MYSQL_RANDOM_ROOT_PASSWORD=yes
volumes:
- mysql-data:/var/lib/mysql
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
environment:
MONGO_INITDB_DATABASE: testdb
networks:
- portabase
sqlite:
container_name: db-sqlite
image: keinos/sqlite3
volumes:
- sqlite-data:/workspace/data
working_dir: /workspace
command: tail -f /dev/null
stdin_open: true
tty: true
db-redis:
image: redis:latest
container_name: db-redis
ports:
- "6378:6379"
volumes:
- redis-data:/data
command: [ "redis-server", "--appendonly", "yes" ]
networks:
- portabase
db-redis-auth:
image: redis:latest
container_name: db-redis-auth
ports:
- "6380:6379"
volumes:
- redis-data-auth:/data
environment:
- REDIS_PASSWORD=supersecurepassword
command: [ "redis-server", "--requirepass", "supersecurepassword", "--appendonly", "yes" ]
networks:
- portabase
db-valkey:
image: valkey/valkey
container_name: db-valkey
environment:
- ALLOW_EMPTY_PASSWORD=yes
ports:
- '6381:6379'
volumes:
- valkey-data:/data
networks:
- portabase
db-valkey-auth:
image: valkey/valkey
container_name: db-valkey-auth
command: >
--requirepass "supersecurepassword"
ports:
- '6382:6379'
volumes:
- valkey-data-auth:/data
networks:
- portabase
db-firebird:
image: firebirdsql/firebird
container_name: db-firebird
restart: always
environment:
- FIREBIRD_ROOT_PASSWORD=fake_root_password
- FIREBIRD_USER=alice
- FIREBIRD_PASSWORD=fake_password
- FIREBIRD_DATABASE=mirror.fdb
- FIREBIRD_DATABASE_DEFAULT_CHARSET=UTF8
volumes:
- firebird-data:/var/lib/firebird/data
ports:
- "3060:3050"
networks:
- portabase
db-mssql:
container_name: db-mssql
image: mcr.microsoft.com/azure-sql-edge:latest
ports:
- "1433:1433"
environment:
ACCEPT_EULA: "Y"
MSSQL_SA_PASSWORD: "Portabase!Strong1"
volumes:
- mssql-data:/var/opt/mssql
networks:
- portabase
healthcheck:
test: ["CMD-SHELL", "cat /proc/net/tcp6 | grep -q '059901' || exit 1"]
interval: 10s
timeout: 5s
retries: 20
volumes:
postgres-data:
postgres-data-2:
mariadb-data:
mysql-data:
mongodb-data:
mongodb-data-auth:
sqlite-data:
redis-data:
redis-data-auth:
valkey-data:
valkey-data-auth:
firebird-data:
mssql-data:
networks:
portabase:
name: portabase_network
external: true
+8 -53
View File
@@ -1,69 +1,24 @@
services:
rust-app:
# build:
# context: .
# dockerfile: docker/Dockerfile
# target: prod
image: portabase/agent:latest
# platform: linux/arm64
build:
context: .
dockerfile: docker/Dockerfile
target: prod
# image: portabase/agent:latest
container_name: rust-prod
volumes:
- ./databases.json:/config/config.json
- /var/run/docker.sock:/var/run/docker.sock
environment:
# APP_ENV: production
LOG: info
TZ: "Europe/Paris"
# DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGZmMDE4NTQtYjJhMS00ZTE0LTkwMjctZTJiOWIxZjQ1YzdlIiwicHVibGljS2V5IjoiLS0tLS1CRUdJTiBSU0EgUFVCTElDIEtFWS0tLS0tXG5NSUlCQ2dLQ0FRRUE5TWV4M2pmdnVLdFB5YU1ERnh2Ulp2dmd3YkRJQ2JzQi81Wll5NDNSVVRBaXZRYjJiSDdYXG5qRHBQd1lJeCs4UFBrbHlRbDVMQzV1UWZEaCs4SVd4OG1LZ3FvMXpWMkdiZXdGbEdEWFYxVEdyU1ZEU25aSWR4XG52bWdYc29EeXhVMlJvWUFUMS9YMWxuc2YxenZKdkFMTkhXdEhRdk42SjVDZTFSMmFsendVRGFEVXlJNzRmSldQXG5tNTh0SDMrYklXL0VVTXdjaWNxM0oySWw3Vm9KNkZNUHJQL1ZSOWEvdFF1SU1qa200MXpFY2NscExPa2luRkxuXG54NmVUWkFSZUpya2UrbnRvZ2t4TGEyRWV5a1lUNzB4V3hKNWp5ZExBVnRvNkkyQlVLVVJoTkowTUFaU29NYUtvXG5iMGJRcnY1UzExZWllMnMrT2I3aTYzSFpkVUx0UmV1MVJ3SURBUUFCXG4tLS0tLUVORCBSU0EgUFVCTElDIEtFWS0tLS0tXG4ifQ"
#DATABASES_CONFIG_FILE: "config.toml"
EDGE_KEY: "<to complete>"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
db-mongodb-auth:
container_name: db-mongodb-auth
image: mongo:latest
ports:
- "27082:27017"
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: rootpassword
MONGO_INITDB_DATABASE: testdbauth
command: mongod --auth
networks:
- portabase
volumes:
- mongodb-data-auth:/data/db
healthcheck:
test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
db-mongodb:
container_name: db-mongodb
image: mongo:latest
ports:
- "27083:27017"
volumes:
- mongodb-data:/data/db
healthcheck:
test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
interval: 5s
timeout: 5s
retries: 10
environment:
MONGO_INITDB_DATABASE: testdb
networks:
- portabase
volumes:
mongodb-data:
mongodb-data-auth:
networks:
portabase:
name: portabase_network
+22
View File
@@ -0,0 +1,22 @@
services:
agent-test:
build:
context: .
dockerfile: docker/Dockerfile
target: dev
working_dir: /app
container_name: agent-test
volumes:
- .:/app
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
- /var/run/docker.sock:/var/run/docker.sock
environment:
APP_ENV: test
LOG: debug
TZ: Europe/Paris
EDGE_KEY: ""
volumes:
cargo-registry:
cargo-git:
+17 -91
View File
@@ -7,117 +7,43 @@ services:
container_name: rust-dev
volumes:
- .:/app
- ./databases.json:/config/config.json
# - ./databases.toml:/config/config.toml
- cargo-registry:/usr/local/cargo/registry
- cargo-git:/usr/local/cargo/git
# - cargo-target:/app/target
# - sqlite-data:/sqlite-data/workspace/data
# - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
- ./databases.json:/config/config.json
#- ./databases.toml:/config/config.toml
- /var/run/docker.sock:/var/run/docker.sock
# - cargo-target:/app/target
- databases_sqlite-data:/sqlite-data/workspace/data
- ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data
# - /bigdisk:/scratch
environment:
APP_ENV: development
LOG: debug
TZ: "Europe/Paris"
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNjI1MDQzY2YtN2MwMC00M2M4LWJjYzktZDM1MTk5ODk2ZGNkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
# TMPDIR: /scratch
EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZDY4MzU2MTQtNzE2NC00OTQ4LWJlZjMtMTlkZDc5NGQzYmRhIiwibWFzdGVyS2V5QjY0IjoiV2NiM0pQQkVTaFBjRjg5UXZwRVJuamU4NGZmak1kNm4vS2dJOUpjMCtmVT0ifQ=="
#CHUNK_SIZE_MB: "1"
#POOLING: 1
#DATABASES_CONFIG_FILE: "config.toml"
extra_hosts:
- "localhost:host-gateway"
networks:
- portabase
db-postgres:
container_name: db-postgres
image: postgres:17-alpine
ports:
- "5436:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
environment:
- POSTGRES_DB=devdb
- POSTGRES_USER=devuser
- POSTGRES_PASSWORD=changeme
networks:
- portabase
#
# db-mariadb:
# container_name: db-mariadb
# image: mariadb:latest
# ports:
# - "3311:3306"
# environment:
# - MYSQL_DATABASE=mariadb
# - MYSQL_USER=mariadb
# - MYSQL_PASSWORD=changeme
# - MYSQL_RANDOM_ROOT_PASSWORD=yes
# volumes:
# - mariadb-data:/var/lib/mysql
# networks:
# - portabase
#
#
# db-mongodb-auth:
# container_name: db-mongodb-auth
# image: mongo:latest
# ports:
# - "27082:27017"
# environment:
# MONGO_INITDB_ROOT_USERNAME: root
# MONGO_INITDB_ROOT_PASSWORD: rootpassword
# MONGO_INITDB_DATABASE: testdbauth
# command: mongod --auth
# networks:
# - portabase
# volumes:
# - mongodb-data-auth:/data/db
# healthcheck:
# test: [ "CMD", "mongo", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
#
# db-mongodb:
# container_name: db-mongodb
# image: mongo:latest
# ports:
# - "27083:27017"
# volumes:
# - mongodb-data:/data/db
# healthcheck:
# test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ]
# interval: 5s
# timeout: 5s
# retries: 10
# environment:
# MONGO_INITDB_DATABASE: testdb
# networks:
# - portabase
# sqlite:
# container_name: db-sqlite
# image: keinos/sqlite3
# volumes:
# - sqlite-data:/workspace/data
# working_dir: /workspace
# command: tail -f /dev/null
# stdin_open: true
# tty: true
cpus: "1.50"
mem_limit: 4g
memswap_limit: 4g
pids_limit: 512
volumes:
cargo-registry:
cargo-git:
# cargo-target:
postgres-data:
# mariadb-data:
# mongodb-data:
# mongodb-data-auth:
# sqlite-data:
# cargo-target:
databases_sqlite-data:
external: true
networks:
portabase:
name: portabase_network
external: true
# docker network create portabase_network
+49 -2
View File
@@ -1,7 +1,15 @@
# =========================
# MySQL client tools
# =========================
FROM mysql:8.4 AS mysql-client-tools
RUN mkdir -p /mysql-exports/bin /mysql-exports/lib \
&& cp /usr/bin/mysqldump /mysql-exports/bin/ \
&& find /usr/lib -name "libmysqlclient.so.21*" -exec cp {} /mysql-exports/lib/ \;
# =========================
# Base image (shared)
# =========================
FROM rust:1.92.0 AS base
FROM rust:1.94.0 AS base
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
pkg-config \
@@ -16,9 +24,26 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
curl \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
&& /tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/dotnet \
&& rm /tmp/dotnet-install.sh \
&& /usr/local/dotnet/dotnet tool install --global microsoft.sqlpackage
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
RUN ARCH=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/') \
&& curl -sSL "https://github.com/microsoft/go-sqlcmd/releases/latest/download/sqlcmd-linux-${ARCH}.tar.bz2" \
| tar -xjf - -C /usr/local/bin sqlcmd \
&& chmod +x /usr/local/bin/sqlcmd
ARG TARGETARCH
# =========================
@@ -44,12 +69,20 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then \
rm -rf /tmp/pg-x64 /tmp/pg-arm && \
chmod +x /usr/lib/postgresql/*/bin/*
# =========================
# MongoDB client binaries
# =========================
COPY assets/tools/${TARGETARCH}/mongodb/ /usr/local/mongodb/
RUN chmod +x /usr/local/mongodb/bin/*
# =========================
# MySQL real mysqldump binary
# =========================
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
WORKDIR /app
# =========================
@@ -58,6 +91,7 @@ WORKDIR /app
FROM base AS dev
RUN cargo install cargo-watch
RUN cargo install grcov --locked
COPY Cargo.toml Cargo.lock ./
RUN mkdir src && echo "fn main() {}" > src/main.rs
@@ -92,10 +126,19 @@ RUN apt-get update && apt-get install -y \
libreadline8 \
libncurses6 \
zlib1g \
curl \
mariadb-client \
sqlite3 \
redis-tools \
valkey \
firebird3.0-utils \
&& rm -rf /var/lib/apt/lists/*
ENV DOTNET_ROOT=/usr/local/dotnet
RUN curl -sSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh \
&& chmod +x /tmp/dotnet-install.sh \
&& /tmp/dotnet-install.sh --channel 8.0 --runtime dotnet --install-dir /usr/local/dotnet \
&& rm /tmp/dotnet-install.sh
WORKDIR /app
@@ -106,8 +149,12 @@ RUN chmod +x /entrypoint.sh
COPY --from=base /usr/lib/postgresql/ /usr/lib/postgresql/
COPY --from=base /usr/local/mongodb/bin/ /usr/local/mongodb/bin/
COPY --from=base /root/.dotnet/tools/ /root/.dotnet/tools/
COPY --from=mysql-client-tools /mysql-exports/bin/mysqldump /usr/local/bin/mysqldump
COPY --from=mysql-client-tools /mysql-exports/lib/ /usr/local/lib/
RUN chmod +x /usr/local/bin/mysqldump && ldconfig
ENV PATH="$PATH:/usr/local/dotnet:/root/.dotnet/tools"
ENV APP_ENV=production
CMD ["/entrypoint.sh"]
+44
View File
@@ -0,0 +1,44 @@
#!/bin/bash
set -euo pipefail
check_docker() {
if ! docker info > /dev/null 2>&1; then
echo "Docker is not running. Attempting to start Docker..."
if [[ "$OSTYPE" == "darwin"* ]]; then
open -a Docker
echo "Waiting for Docker to start..."
until docker info > /dev/null 2>&1; do
sleep 2
done
elif command -v systemctl >/dev/null 2>&1; then
sudo systemctl start docker
else
echo "Cannot start Docker automatically. Please start Docker manually."
exit 1
fi
else
echo "Docker is running."
fi
}
check_network() {
local network_name="portabase_network"
if ! docker network ls --format '{{.Name}}' | grep -q "^${network_name}$"; then
echo "Docker network '${network_name}' not found. Creating..."
docker network create "${network_name}"
else
echo "Docker network '${network_name}' already exists."
fi
}
check_docker
check_network
echo "Stopping old database containers..."
docker compose -f ./docker-compose.databases.yml down
echo "Starting database containers..."
docker compose -f ./docker-compose.databases.yml up -d
echo "Starting main services..."
docker compose -f ./docker-compose.yml up
+10 -3
View File
@@ -34,16 +34,23 @@ if [ -n "$TZ" ]; then
fi
fi
REDIS_PORT=65515
echo "[entrypoint] APP_ENV=$APP_ENV"
echo "[entrypoint] Starting Redis..."
redis-server --daemonize yes
redis-server --port $REDIS_PORT --daemonize yes
echo "[entrypoint] Waiting for Redis to be ready..."
until redis-cli ping >/dev/null 2>&1; do
MAX_RETRIES=20
COUNT=0
until redis-cli -h localhost -p "$REDIS_PORT" ping >/dev/null 2>&1 ; do
COUNT=$((COUNT+1))
if [ $COUNT -ge $MAX_RETRIES ]; then
echo "[ERROR] Redis did not start after $MAX_RETRIES attempts"
exit 1
fi
echo "[entrypoint] Redis not ready, sleeping 1s..."
sleep 1
done
echo "[entrypoint] Redis is ready"
+17
View File
@@ -0,0 +1,17 @@
.DS_Store
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
*.swp
*.bak
*.tmp
*.orig
*~
.project
.idea/
*.tmproj
.vscode/
+29
View File
@@ -0,0 +1,29 @@
apiVersion: v2
name: portabase-agent
description: Helm chart for Portabase Agent
type: application
version: 0.0.0
appVersion: "latest"
keywords:
- postgresql
- mariadb
- mongodb
- mysql
- sqlite
- backup
- database
- restore
- agent
home: https://github.com/Portabase/agent
sources:
- https://github.com/Portabase/agent
- https://github.com/Portabase/agent/tree/main/helm
maintainers:
- name: Charles Gauthereau
url: https://github.com/RambokDev
- name: Killian Larcher
url: https://github.com/KillianLarcher
icon: https://raw.githubusercontent.com/Portabase/agent/main/.github/assets/logo.png
+59
View File
@@ -0,0 +1,59 @@
# Development Notes
## Check that Kubernetes is reachable locally
```bash
kubectl get nodes
```
## Install the local Portabase Agent Helm chart
```bash
helm install portabase-agent . \
--set env.EDGE_KEY=<your-edge-key>
```
## Check the pods
```bash
kubectl get pods
```
## Check the services
```bash
kubectl get svc
```
## To update .env variables or JSON config:
```bash
kubectl rollout restart deployment portabase-agent
```
## Install or upgrade the Helm chart
```bash
helm upgrade portabase-agent . \
--reuse-values \
--set env.EDGE_KEY="NEW_EDGE_KEY"
```
## Rollout to restart
```bash
kubectl rollout restart deployment portabase-agent
```
## List pods to get the pod name
```bash
kubectl get pods -l app=portabase-agent
```
## Get logs for the pod
```bash
kubectl logs portabase-agent-6f7d4f5c6b-abc12
```
## Uninstall Agent
``` bash
helm uninstall portabase-agent
```
+7
View File
@@ -0,0 +1,7 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: portabase-agent-config
data:
config.json: |
{{ .Values.volume.configFile.content | nindent 4 }}
+62
View File
@@ -0,0 +1,62 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: portabase-agent
labels:
app: portabase-agent
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: portabase-agent
template:
metadata:
labels:
app: portabase-agent
spec:
hostAliases:
{{- range .Values.network.hostAliases }}
- ip: {{ .ip }}
hostnames:
{{- range .hostnames }}
- {{ . }}
{{- end }}
{{- end }}
containers:
- name: portabase-agent
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
envFrom:
- configMapRef:
name: portabase-agent-env
volumeMounts:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
- name: config
mountPath: /config/config.json
subPath: config.json
readOnly: true
# uses hostPath
{{- else }}
- name: config
mountPath: /config/config.json
subPath: config.json
# uses ConfigMap content
{{- end }}
{{- end }}
volumes:
{{- if .Values.volume.configFile.enabled }}
{{- if .Values.volume.configFile.hostPath }}
- name: config
hostPath:
path: {{ .Values.volume.configFile.hostPath }}
type: File
{{- else }}
- name: config
configMap:
name: portabase-agent-config
items:
- key: config.json
path: config.json
{{- end }}
{{- end }}
+10
View File
@@ -0,0 +1,10 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: portabase-agent-env
data:
EDGE_KEY: {{ .Values.env.EDGE_KEY | quote }}
TZ: {{ .Values.env.TZ | quote }}
POLLING: {{ .Values.env.POLLING | quote }}
APP_ENV: {{ .Values.env.APP_ENV | quote }}
LOG: {{ .Values.env.LOG | quote }}
+48
View File
@@ -0,0 +1,48 @@
replicaCount: 1
image:
repository: portabase/agent
tag: latest
pullPolicy: IfNotPresent
env:
EDGE_KEY: "your_edge_key_here"
TZ: "UTC"
POLLING: "5"
APP_ENV: "production"
LOG: "info"
resources:
limits:
cpu: 500m
memory: 512Mi
requests:
cpu: 250m
memory: 256Mi
volume:
configFile:
enabled: true
hostPath: "" # Use host file if set, otherwise use `content`
content: | # JSON content for config.json if no hostPath
{
"databases": [
{
"name": "my-site-prod (readable name)",
"database": "devdb",
"type": "postgresql",
"host": "localhost",
"port": 5432,
"username": "admin_prod",
"password": "super_secure_password",
"generated_id": "550e8400-e29b-41d4-a716-446655440000"
}
]
}
network:
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "localhost"
+79
View File
@@ -0,0 +1,79 @@
set dotenv-load := true
set shell := ["bash", "-cu"]
CLUSTER_SCRIPT := "docker/entrypoints/app-dev-entrypoint.sh"
up:
bash {{CLUSTER_SCRIPT}}
seed-mongo:
echo "Seeding MongoDB..."
bash ./scripts/mongo/seed-mongo.sh
seed-mongo-auth:
echo "Seeding MongoDB with auth..."
bash ./scripts/mongo/seed-mongo.sh auth
seed-mysql:
echo "Seeding MySQL..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-mysql.sql
seed-mysql-1gb:
echo "Seeding MySQL (1GB)..."
mysql -h 127.0.0.1 -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DB" < ./scripts/mysql/seed-1gb.sql
seed-postgres:
echo "Seeding Postgres..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed.sql
seed-postgres-1gb:
echo "Seeding Postgres (1GB)..."
docker exec -i -e PGPASSWORD="$PG_PASSWORD" "$PG_CONTAINER" \
psql -U "$PG_USER" -d "$PG_DB" < ./scripts/postgres/seed-1gb.sql
seed-sqlite:
echo "Seeding SQLite..."
bash ./scripts/sqlite/seed.sh
echo "Done"
seed-firebird:
echo "Seeding Firebird..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed.sql
echo "Verifying Firebird tables..."
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
seed-firebird-large:
echo "Seeding Firebird..."
docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb < ./scripts/firebird/seed-large.sql
echo "Verifying Firebird tables..."
echo "SELECT RDB\$RELATION_NAME FROM RDB\$RELATIONS WHERE RDB\$SYSTEM_FLAG = 0 AND RDB\$VIEW_BLR IS NULL;" \
| docker exec -i db-firebird isql -user alice -password fake_password /var/lib/firebird/data/mirror.fdb
seed-mssql:
echo "Seeding MSSQL..."
docker exec -i rust-dev sqlcmd -S "db-mssql,1433" -U sa -P "$MSSQL_SA_PASSWORD" -N disable -i /app/scripts/mssql/seed.sql
echo "Done"
seed-all:
just seed-mongo
just seed-mysql
just seed-postgres
just seed-postgres-1gb
just seed-sqlite
just seed-mongo
just seed-firebird
just seed-mssql
test:
echo "Build test image (with grcov included)"
docker compose -f docker-compose.test.yml build agent-test
echo "Start agent-test container"
docker compose -f docker-compose.test.yml up -d agent-test
echo "Run tests inside container"
docker compose -f docker-compose.test.yml exec -e CARGO_INCREMENTAL -e RUSTFLAGS -e LLVM_PROFILE_FILE agent-test bash -c "cargo test --verbose && sync"
echo "Down volumes tests databases"
docker compose -f docker-compose.test.yml down --volumes
+4
View File
@@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI
```bash
docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';"
```
```bash
docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test"
```
+259
View File
@@ -0,0 +1,259 @@
SET SQL DIALECT 3;
SET BAIL ON;
SET AUTODDL OFF;
CREATE TABLE users (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
email VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(255),
payload BLOB SUB_TYPE TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
COMMIT;
INSERT INTO users (email, name, payload)
VALUES ('alice@example.com', 'Alice', 'Alice seed data');
INSERT INTO users (email, name, payload)
VALUES ('bob@example.com', 'Bob', 'Bob seed data');
COMMIT;
/*
* Each procedure call generates approximately 128 MiB:
*
* 128 rows
* × 128 chunks per row
* × 8191 bytes per chunk
* = approximately 128 MiB
*
* 40 calls = approximately 5 GiB.
*/
SET TERM ^;
CREATE PROCEDURE seed_users_batch (
p_rows INTEGER,
p_chunks_per_row INTEGER
)
AS
DECLARE VARIABLE v_row_index INTEGER;
DECLARE VARIABLE v_chunk_index INTEGER;
DECLARE VARIABLE v_uuid VARCHAR(36);
DECLARE VARIABLE v_chunk VARCHAR(8191);
DECLARE VARIABLE v_payload BLOB SUB_TYPE TEXT;
BEGIN
v_row_index = 0;
WHILE (v_row_index < p_rows) DO
BEGIN
v_payload = NULL;
v_chunk_index = 0;
WHILE (v_chunk_index < p_chunks_per_row) DO
BEGIN
/*
* Generate a different chunk to avoid producing a completely
* uniform BLOB.
*/
v_chunk = RPAD(
UUID_TO_CHAR(GEN_UUID()),
8191,
UUID_TO_CHAR(GEN_UUID())
);
v_payload = BLOB_APPEND(v_payload, v_chunk);
v_chunk_index = v_chunk_index + 1;
END
v_uuid = UUID_TO_CHAR(GEN_UUID());
INSERT INTO users (
email,
name,
payload
)
VALUES (
:v_uuid || '@example.test',
'Seed User ' || :v_uuid,
:v_payload
);
v_row_index = v_row_index + 1;
END
END^
SET TERM ;^
/* Batch 01 — approximately 128 MiB */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 02 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 03 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 04 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 05 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 06 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 07 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 08 — approximately 1 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 09 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 10 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 11 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 12 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 13 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 14 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 15 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 16 — approximately 2 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 17 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 18 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 19 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 20 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 21 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 22 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 23 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 24 — approximately 3 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 25 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 26 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 27 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 28 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 29 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 30 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 31 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 32 — approximately 4 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 33 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 34 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 35 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 36 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 37 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 38 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 39 */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
/* Batch 40 — approximately 5 GiB total */
EXECUTE PROCEDURE seed_users_batch(128, 128);
COMMIT;
DROP PROCEDURE seed_users_batch;
COMMIT;
SELECT
COUNT(*) AS user_count,
CAST(SUM(OCTET_LENGTH(payload)) / 1073741824.0 AS DECIMAL(18, 2))
AS payload_size_gib
FROM users;
COMMIT;
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE users (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
email VARCHAR(255) NOT NULL UNIQUE,
name VARCHAR(255),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
COMMIT;
+20
View File
@@ -0,0 +1,20 @@
IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = N'myappdb')
CREATE DATABASE [myappdb];
GO
USE [myappdb];
GO
IF OBJECT_ID('users', 'U') IS NULL
CREATE TABLE users (
id INT IDENTITY(1,1) PRIMARY KEY,
email NVARCHAR(255) NOT NULL UNIQUE,
name NVARCHAR(255),
created_at DATETIME DEFAULT GETDATE()
);
GO
INSERT INTO users (email, name) VALUES ('alice@example.com', 'Alice');
INSERT INTO users (email, name) VALUES ('bob@example.com', 'Bob');
INSERT INTO users (email, name) VALUES ('charlie@example.com', 'Charlie');
GO
+14
View File
@@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
SEED_VALUE="${SEED:-}"
if [ "$SEED_VALUE" = "big" ]; then
SQLITE_SEED_FILE="./scripts/sqlite/seed-big.sql"
else
SQLITE_SEED_FILE="./scripts/sqlite/seed.sql"
fi
docker exec -u 0 db-sqlite sh -c "chmod -R 777 /workspace/data"
docker exec -u 0 db-sqlite sh -c "touch /workspace/data/app.db"
docker exec -i db-sqlite sh -c "sqlite3 /workspace/data/app.db" < "$SQLITE_SEED_FILE"
+44 -14
View File
@@ -2,13 +2,16 @@
use crate::core::context::Context;
use crate::services::backup::BackupService;
use crate::services::config::ConfigService;
use crate::services::config::{ConfigService, DatabaseConfig};
use crate::services::cron::CronService;
use crate::services::status::StatusService;
use crate::utils::common::BackupMethod;
use std::sync::Arc;
use tracing::info;
use crate::services::dashboard_config::{collect_configs, load_cache, merge, persist_cache};
use crate::services::restore::RestoreService;
use crate::services::status::StatusService;
use crate::settings::CONFIG;
use crate::utils::common::BackupMethod;
use std::path::PathBuf;
use std::sync::Arc;
use tracing::{error, info, warn};
pub struct Agent {
ctx: Arc<Context>,
@@ -17,6 +20,8 @@ pub struct Agent {
cron_service: CronService,
backup_service: BackupService,
restore_service: RestoreService,
dashboard_cache: Vec<DatabaseConfig>,
cache_path: PathBuf,
}
impl Agent {
@@ -28,6 +33,9 @@ impl Agent {
let backup_service = BackupService::new(ctx.clone());
let restore_service = RestoreService::new(ctx.clone());
let cache_path = PathBuf::from(&CONFIG.data_path).join("dashboard_databases.json");
let dashboard_cache = load_cache(&cache_path);
Agent {
ctx,
config_service,
@@ -35,30 +43,52 @@ impl Agent {
cron_service,
backup_service,
restore_service,
dashboard_cache,
cache_path,
}
}
pub async fn run(&mut self, method: BackupMethod) -> Result<(), Box<dyn std::error::Error>> {
let config = self.config_service.load(None)?;
let ping_result = self.status_service.ping(&config.databases).await?;
let local = self.config_service.load_optional(None);
let merged_in = merge(&local.databases, &self.dashboard_cache);
let ping_result = self.status_service.ping(&merged_in.databases).await?;
self.dashboard_cache = collect_configs(&ping_result);
if let Err(e) = persist_cache(&self.cache_path, &self.dashboard_cache) {
error!("Failed to persist dashboard cache: {e}");
}
let merged = merge(&local.databases, &self.dashboard_cache);
for db in ping_result.databases.iter() {
let Some(database) = merged
.databases
.iter()
.find(|cfg_db| cfg_db.generated_id == db.generated_id)
else {
warn!("No config for returned database {}; skipping", db.generated_id);
continue;
};
info!(
"Generated Id: {} | backup action: {} | restore action: {}",
db.generated_id, db.data.backup.action, db.data.restore.action
"Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}",
db.generated_id, db.data.backup.action, db.data.restore.action, database.name,
);
let _ = self.cron_service.sync(db).await;
if db.data.backup.action {
let _ = self
.backup_service
.dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt)
.dispatch(
&db.generated_id,
&merged,
method.clone(),
&db.storages,
db.encrypt,
)
.await;
} else if db.data.restore.action {
let _ = self
.restore_service
.dispatch(db, &config)
.await;
let _ = self.restore_service.dispatch(db, &merged).await;
}
}
+1 -1
View File
@@ -35,7 +35,7 @@ impl Context {
panic!("Cannot initialize AgentContext due to invalid EDGE_KEY");
}
};
let server_url = format!("{}/api", edge_key.server_url);
let api_client = ApiClient::new(server_url);
+1 -1
View File
@@ -1,2 +1,2 @@
pub mod context;
pub mod agent;
pub mod context;
+68
View File
@@ -0,0 +1,68 @@
use crate::domain::docker_volume::docker::{
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use bollard::query_parameters::DownloadFromContainerOptions;
use futures_util::StreamExt;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
pub async fn run(cfg: DatabaseConfig, backup_dir: PathBuf, logger: Arc<JobLogger>) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
futures::executor::block_on(async move {
logger.log("info", format!("Starting docker-volume backup for {}", cfg.name));
let docker = client()?;
let image = resolve_helper_image(&docker).await?;
logger.log("debug", format!("Helper image: {image}"));
if let Some(name) = &cfg.container_name {
logger.log("info", format!("Stopping container {name} for consistent backup"));
stop_container(&docker, name).await?;
}
let result = async {
let helper = create_helper(&docker, &image, &cfg.volume_name, &cfg.generated_id, true, None).await?;
let file_path = backup_dir.join(format!("{}.tar", cfg.generated_id));
let start = Instant::now();
let dl_opts = DownloadFromContainerOptions { path: "/vol".to_string() };
let mut stream = docker.download_from_container(&helper.id, Some(dl_opts));
let mut out = File::create(&file_path)
.await
.with_context(|| format!("Failed to create backup file {}", file_path.display()))?;
let mut bytes_written: u64 = 0;
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("Error streaming volume archive from Docker")?;
bytes_written += chunk.len() as u64;
out.write_all(&chunk).await?;
}
out.flush().await?;
let duration_ms = start.elapsed().as_millis() as f64;
logger.log_command("docker download_from_container", None, Some(0), Some(duration_ms));
logger.log("info", format!("Volume backup wrote {bytes_written} bytes to {}", file_path.display()));
remove_helper(&docker, &helper.id).await;
anyhow::Ok(file_path)
}
.await;
if let Some(name) = &cfg.container_name {
if let Err(e) = start_container(&docker, name).await {
logger.log("error", format!("Failed to restart container {name}: {e}"));
}
}
result
})
})
.await?
}
+45
View File
@@ -0,0 +1,45 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
pub struct DockerVolumeDatabase {
cfg: DatabaseConfig,
}
impl DockerVolumeDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for DockerVolumeDatabase {
fn file_extension(&self) -> &'static str {
".tar"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+169
View File
@@ -0,0 +1,169 @@
#![allow(dead_code)]
use anyhow::{Context, Result};
use bollard::Docker;
use bollard::models::{ContainerCreateBody, HostConfig};
use bollard::query_parameters::{
CreateContainerOptions, InspectContainerOptions, ListContainersOptions,
RemoveContainerOptions, StartContainerOptions, StopContainerOptions,
};
use std::collections::HashMap;
use tracing::{info, warn};
use uuid::Uuid;
pub const EPHEMERAL_LABEL: &str = "io.portabase.ephemeral";
const HELPER_MOUNT: &str = "/vol";
pub fn client() -> Result<Docker> {
Docker::connect_with_defaults().context("Failed to connect to Docker daemon socket")
}
pub fn parse_container_id(mountinfo: &str, cgroup: &str) -> Option<String> {
for src in [mountinfo, cgroup] {
for line in src.lines() {
for marker in ["/containers/", "/docker/"] {
if let Some(idx) = line.find(marker) {
let rest = &line[idx + marker.len()..];
let id: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
if id.len() >= 64 {
return Some(id[..64].to_string());
}
}
}
}
}
None
}
pub async fn resolve_helper_image(docker: &Docker) -> Result<String> {
if let Ok(img) = std::env::var("PORTABASE_HELPER_IMAGE") {
if !img.trim().is_empty() {
return Ok(img);
}
}
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").unwrap_or_default();
let cgroup = std::fs::read_to_string("/proc/self/cgroup").unwrap_or_default();
let id = parse_container_id(&mountinfo, &cgroup).context(
"Could not determine own container id; set PORTABASE_HELPER_IMAGE to a locally-present image",
)?;
let info = docker
.inspect_container(&id, None::<InspectContainerOptions>)
.await
.with_context(|| format!("Failed to inspect self container {id}"))?;
info.image
.context("Self container inspection returned no image reference")
}
pub struct Helper {
pub id: String,
}
pub async fn create_helper(
docker: &Docker,
image: &str,
volume_name: &str,
generated_id: &str,
read_only: bool,
cmd: Option<Vec<String>>,
) -> Result<Helper> {
let bind = format!(
"{volume_name}:{HELPER_MOUNT}{}",
if read_only { ":ro" } else { "" }
);
let mut labels = HashMap::new();
labels.insert(EPHEMERAL_LABEL.to_string(), "true".to_string());
labels.insert("com.docker.compose.project".to_string(), String::new());
labels.insert("com.docker.compose.service".to_string(), String::new());
labels.insert("com.docker.compose.oneoff".to_string(), String::new());
let name = format!(
"portabase-vol-{generated_id}-{}",
&Uuid::new_v4().to_string()[..8]
);
let body = ContainerCreateBody {
image: Some(image.to_string()),
cmd,
labels: Some(labels),
host_config: Some(HostConfig {
binds: Some(vec![bind]),
auto_remove: Some(false),
..Default::default()
}),
..Default::default()
};
let opts = CreateContainerOptions {
name: Some(name),
..Default::default()
};
let res = docker
.create_container(Some(opts), body)
.await
.with_context(|| format!("Failed to create helper container for volume {volume_name}"))?;
Ok(Helper { id: res.id })
}
pub async fn remove_helper(docker: &Docker, id: &str) {
let stop_opts = StopContainerOptions {
t: Some(2),
..Default::default()
};
let _ = docker.stop_container(id, Some(stop_opts)).await;
if let Ok(info) = docker
.inspect_container(id, None::<InspectContainerOptions>)
.await
{
let name = info.name.unwrap_or_default();
let name = name.trim_start_matches('/');
let code = info.state.and_then(|s| s.exit_code).unwrap_or_default();
info!("Helper container {name} exited with code {code}");
}
let opts = RemoveContainerOptions {
force: true,
..Default::default()
};
if let Err(e) = docker.remove_container(id, Some(opts)).await {
warn!("Failed to remove helper container {id}: {e}");
}
}
pub async fn stop_container(docker: &Docker, name: &str) -> Result<()> {
docker
.stop_container(name, None::<StopContainerOptions>)
.await
.with_context(|| format!("Failed to stop container {name}"))
}
pub async fn start_container(docker: &Docker, name: &str) -> Result<()> {
docker
.start_container(name, None::<StartContainerOptions>)
.await
.with_context(|| format!("Failed to start container {name}"))
}
pub async fn sweep_ephemeral(docker: &Docker) -> Result<usize> {
let mut filters = HashMap::new();
filters.insert("label".to_string(), vec![format!("{EPHEMERAL_LABEL}=true")]);
let opts = ListContainersOptions {
all: true,
filters: Some(filters),
..Default::default()
};
let list = docker.list_containers(Some(opts)).await?;
let mut removed = 0;
for c in list {
if let Some(id) = c.id {
remove_helper(docker, &id).await;
removed += 1;
}
}
Ok(removed)
}
+5
View File
@@ -0,0 +1,5 @@
pub mod backup;
pub mod database;
pub mod docker;
pub mod ping;
pub mod restore;
+12
View File
@@ -0,0 +1,12 @@
use crate::domain::docker_volume::docker::client;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
let docker = client()?;
match docker.inspect_volume(&cfg.volume_name).await {
Ok(_) => Ok(true),
Err(bollard::errors::Error::DockerResponseServerError { status_code: 404, .. }) => Ok(false),
Err(e) => Err(e.into()),
}
}
+102
View File
@@ -0,0 +1,102 @@
use crate::domain::docker_volume::docker::{
client, create_helper, remove_helper, resolve_helper_image, start_container, stop_container,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use bollard::exec::StartExecResults;
use bollard::models::ExecConfig;
use bollard::query_parameters::UploadToContainerOptions;
use futures_util::StreamExt;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, archive: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
futures::executor::block_on(async move {
logger.log("info", format!("Starting docker-volume restore for {}", cfg.name));
let docker = client()?;
let image = resolve_helper_image(&docker).await?;
logger.log("debug", format!("Restore archive: {}", archive.display()));
if let Some(name) = &cfg.container_name {
logger.log("info", format!("Stopping container {name} for restore"));
stop_container(&docker, name).await?;
}
let result = async {
let helper = create_helper(
&docker,
&image,
&cfg.volume_name,
&cfg.generated_id,
false,
Some(vec![
"sh".into(),
"-c".into(),
"trap 'exit 0' TERM; sleep 2147483647 & wait".into(),
]),
)
.await?;
start_container(&docker, &helper.id).await?;
let exec = docker
.create_exec(
&helper.id,
ExecConfig {
cmd: Some(vec![
"sh".to_string(),
"-c".to_string(),
"rm -rf /vol/* /vol/.[!.]* 2>/dev/null || true".to_string(),
]),
attach_stdout: Some(true),
attach_stderr: Some(true),
..Default::default()
},
)
.await
.context("Failed to create wipe exec")?;
if let StartExecResults::Attached { mut output, .. } =
docker.start_exec(&exec.id, None).await.context("Failed to run wipe exec")?
{
while output.next().await.is_some() {}
}
let start = Instant::now();
let file = tokio::fs::File::open(&archive)
.await
.with_context(|| format!("Failed to open {}", archive.display()))?;
let stream = tokio_util::io::ReaderStream::new(file);
let up_opts = UploadToContainerOptions { path: "/".to_string(), ..Default::default() };
docker
.upload_to_container(&helper.id, Some(up_opts), bollard::body_try_stream(stream))
.await
.context("Failed to upload volume archive")?;
let duration_ms = start.elapsed().as_millis() as f64;
logger.log_command("docker upload_to_container", None, Some(0), Some(duration_ms));
remove_helper(&docker, &helper.id).await;
logger.log("info", format!("Volume restore completed for {}", cfg.name));
anyhow::Ok(())
}
.await;
if let Some(name) = &cfg.container_name {
if let Err(e) = start_container(&docker, name).await {
logger.log("error", format!("Failed to restart container {name}: {e}"));
}
}
result
})
})
.await?
}
+25 -5
View File
@@ -1,19 +1,27 @@
use crate::domain::docker_volume::database::DockerVolumeDatabase;
use crate::domain::mongodb::database::MongoDatabase;
use crate::domain::mysql::database::MySQLDatabase;
use crate::domain::postgres::cluster::database::PostgresClusterDatabase;
use crate::domain::postgres::database::PostgresDatabase;
use crate::domain::postgres::{detect_format_from_file, detect_format_from_size};
use crate::domain::redis::database::RedisDatabase;
use crate::domain::sqlite::database::SqliteDatabase;
use crate::domain::valkey::database::ValkeyDatabase;
use crate::domain::firebird::database::FirebirdDatabase;
use crate::domain::mariadb::database::MariaDBDatabase;
use crate::domain::mssql::database::MssqlDatabase;
use crate::services::backup::logger::JobLogger;
use crate::services::config::{DatabaseConfig, DbType};
use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::domain::sqlite::database::SqliteDatabase;
#[async_trait::async_trait]
pub trait Database: Send + Sync {
fn file_extension(&self) -> &'static str;
async fn ping(&self) -> Result<bool>;
async fn backup(&self, backup_dir: &Path) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path) -> Result<()>;
async fn backup(&self, backup_dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf>;
async fn restore(&self, restore_file: &Path, logger: Arc<JobLogger>) -> Result<()>;
}
pub struct DatabaseFactory;
@@ -25,10 +33,16 @@ impl DatabaseFactory {
let format = detect_format_from_size(&cfg).await;
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
}
}
@@ -38,10 +52,16 @@ impl DatabaseFactory {
let format = detect_format_from_file(restore_file);
Arc::new(PostgresDatabase::new(cfg, format))
}
DbType::PostgresqlCluster => Arc::new(PostgresClusterDatabase::new(cfg)),
DbType::Mysql => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MySQLDatabase::new(cfg)),
DbType::Mariadb => Arc::new(MariaDBDatabase::new(cfg)),
DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)),
DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)),
DbType::Redis => Arc::new(RedisDatabase::new(cfg)),
DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)),
DbType::Firebird => Arc::new(FirebirdDatabase::new(cfg)),
DbType::Mssql => Arc::new(MssqlDatabase::new(cfg)),
DbType::DockerVolume => Arc::new(DockerVolumeDatabase::new(cfg)),
}
}
}
+55
View File
@@ -0,0 +1,55 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting Firebird backup for database: {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
logger.log("info", format!("Firebird target: {}{}", db_path, file_path.display()));
let start = Instant::now();
let output = Command::new("gbak")
.arg("-b")
.arg("-g")
.arg("-v")
.arg("-user").arg(&cfg.username)
.arg("-password").arg(&cfg.password)
.arg(&db_path)
.arg(&file_path)
.output()
.with_context(|| format!("Failed to run gbak for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let combined_output = if stderr.is_empty() && stdout.is_empty() {
None
} else {
Some(format!("{}{}", stdout, stderr).trim().to_string())
};
if !output.status.success() {
logger.log_command("gbak", combined_output, Some(exit_code), Some(duration_ms));
anyhow::bail!("Firebird backup failed for {}: {}", cfg.name, stderr);
}
logger.log_command("gbak", combined_output, Some(0), Some(duration_ms));
logger.log("info", format!("Firebird backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+51
View File
@@ -0,0 +1,51 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct FirebirdDatabase {
cfg: DatabaseConfig,
}
impl FirebirdDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for FirebirdDatabase {
fn file_extension(&self) -> &'static str {
".fbk"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+4
View File
@@ -0,0 +1,4 @@
pub mod database;
mod ping;
mod backup;
mod restore;
+57
View File
@@ -0,0 +1,57 @@
use std::process::Stdio;
use tracing::{error, info};
use tokio::io::AsyncWriteExt;
use crate::services::config::DatabaseConfig;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig) -> anyhow::Result<bool> {
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
info!("Running Ping database from {}", db_path);
let mut child = Command::new("isql-fb")
.arg("-q")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&db_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
let query = b"SELECT 1 FROM RDB$DATABASE;\nQUIT;\n";
if let Some(mut stdin) = child.stdin.take() {
stdin.write_all(query).await?;
}
let output = match timeout(Duration::from_secs(5), child.wait_with_output()).await {
Ok(res) => res?,
Err(_) => return Ok(false),
};
if !output.status.success() {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if stderr.to_lowercase().contains("error") {
error!("Error output for firebird: {:?}", output);
return Ok(false);
}
if stdout.contains("1") {
return Ok(true);
}
info!("stdout {}", stdout);
error!("stderr {}", stderr);
Ok(false)
}
+48
View File
@@ -0,0 +1,48 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("debug", format!("Starting Firebird restore for database {}", cfg.name));
let db_path = format!("{}/{}:{}", cfg.host, cfg.port, cfg.database);
logger.log("info", format!("Restore source: {}", restore_file.display()));
logger.log("info", format!("Restore target: {}", db_path));
let start = Instant::now();
let output = Command::new("gbak")
.arg("-c")
.arg("-v")
.arg("-replace_database")
.arg("-user")
.arg(&cfg.username)
.arg("-password")
.arg(&cfg.password)
.arg(&restore_file)
.arg(&db_path)
.output()
.with_context(|| format!("Failed to run gbak restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
logger.log_command("gbak", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("Firebird restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("Firebird restore failed for {}: {}", cfg.name, stderr);
}
logger.log_command("gbak", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Firebird restore completed for {}", cfg.name));
Ok(())
})
.await?
}
+80
View File
@@ -0,0 +1,80 @@
use crate::domain::mariadb::connection::{select_mariadb_path, server_version};
use crate::services::backup::logger::JobLogger;
use crate::domain::mysql::connection::connection_args;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
logger.log("info", format!("MariaDB version detected: {}", v));
v
}
Err(e) => {
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let _mariadb_dump = select_mariadb_path(&version).join("mariadb-dump");
logger.log("debug", format!("Using mariadb-dump at {}", _mariadb_dump.display()));
if let Ok(out) = Command::new("mariadb-dump").arg("--version").output() {
logger.log("debug", format!("mariadb-dump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
}
logger.log("info", format!("Running mariadb-dump for {}", cfg.name));
let start = Instant::now();
let output = Command::new("mariadb-dump")
.args(connection_args(&cfg))
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--single-transaction")
.arg("--quick")
.arg("--skip-lock-tables")
.arg("--no-create-db")
.arg("--skip-add-drop-table")
.arg("--compress")
.arg("--verbose")
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
.arg("--net-buffer-length=16K")
.arg("--default-character-set=utf8mb4")
.arg(&cfg.database)
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mariadb-dump for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log_command("mariadb-dump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("Mariadb backup failed for {}: {}", cfg.name, stderr);
}
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
logger.log_command("mariadb-dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("mariadb-dump completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+41
View File
@@ -0,0 +1,41 @@
use std::path::PathBuf;
use crate::domain::mysql::connection::connection_args;
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use std::process::Command;
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mariadb")
.args(connection_args(cfg))
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!("Version query failed: {}", stderr);
}
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
pub fn select_mariadb_path(version: &str) -> PathBuf {
let mut parts = version.split('.');
let major = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(10);
let minor = parts.next().and_then(|v| v.parse::<u32>().ok()).unwrap_or(0);
if major < 10 || (major == 10 && minor <= 6) {
"/usr/local/mariadb-10.6/bin".into()
} else {
"/usr/local/mariadb-12.1/bin".into()
}
}
+58
View File
@@ -0,0 +1,58 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MariaDBDatabase {
cfg: DatabaseConfig,
}
impl MariaDBDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("MYSQL_PWD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
impl Database for MariaDBDatabase {
fn file_extension(&self) -> &'static str {
".sql"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod backup;
mod connection;
pub mod database;
mod ping;
mod restore;
+24
View File
@@ -0,0 +1,24 @@
use crate::domain::mysql::connection::connection_args;
use crate::services::config::DatabaseConfig;
use std::collections::HashMap;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.args(connection_args(&cfg))
.arg("ping")
.envs(env);
let result = timeout(Duration::from_secs(10), cmd.output()).await;
match result {
Ok(output) => {
let output = output?;
Ok(output.status.success())
}
Err(_) => Ok(false),
}
}
+82
View File
@@ -0,0 +1,82 @@
use crate::services::backup::logger::JobLogger;
use crate::domain::mysql::connection::connection_args;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::fs::File;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting restore for database {}", cfg.name));
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
cfg.database
);
let drop_start = Instant::now();
let drop_output = Command::new("mariadb")
.args(connection_args(&cfg))
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.output()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
if !drop_output.status.success() {
logger.log_command("mariadb", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
logger.log_command("mariadb", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
let start = Instant::now();
let mut child = Command::new("mariadb")
.args(connection_args(&cfg))
.arg("--database")
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("Failed to start MariaDB restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
std::io::copy(&mut file, &mut stdin)
.context("Failed to stream SQL content to MariaDB stdin")?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete MariaDB restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
logger.log_command("mariadb", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MariaDB restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MariaDB restore failed for {}", cfg.name);
}
logger.log_command("mariadb", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
Ok(())
});
handle.await??;
Ok(())
}
+8 -3
View File
@@ -1,6 +1,11 @@
pub mod docker_volume;
pub mod factory;
pub mod postgres;
pub mod mysql;
mod mongodb;
pub mod mysql;
pub mod postgres;
mod redis;
mod sqlite;
mod valkey;
mod mariadb;
mod firebird;
pub mod mssql;
+18 -5
View File
@@ -1,35 +1,48 @@
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting MongoDB backup for database {}", cfg.name);
logger.log("info", format!("Starting MongoDB backup for database {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let mongodump = select_mongo_path().join("mongodump");
let uri = get_mongo_uri(cfg.clone())?;
logger.log("info", format!("Running mongodump for {}", cfg.name));
let start = Instant::now();
let output = Command::new(mongodump)
.arg(format!("--uri={}", uri))
.arg(format!("--archive={}", file_path.display()))
.arg("--gzip")
.arg("--verbose")
.output()
.context("MongoDB backup failed")?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MongoDB backup failed for {}: {}", cfg.name, stderr);
logger.log("error", format!("MongoDB backup failed for {}: {}", cfg.name, stderr));
logger.log_command("mongodump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("MongoDB backup failed for {}: {}", cfg.name, stderr);
}
info!("MongoDB backup completed for {}", cfg.name);
logger.log_command("mongodump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("MongoDB backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
+119 -9
View File
@@ -1,6 +1,13 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use mongodb::Client;
use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
const USERINFO_ENCODE: &AsciiSet = &NON_ALPHANUMERIC
.remove(b'-')
.remove(b'_')
.remove(b'.')
.remove(b'~');
pub async fn connect(cfg: DatabaseConfig) -> Result<Client> {
let uri = get_mongo_uri(cfg)?;
@@ -16,15 +23,118 @@ pub fn select_mongo_path() -> std::path::PathBuf {
}
pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result<String> {
if cfg.username.is_empty() || cfg.password.is_empty() {
Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database))
} else {
Ok(format!(
"mongodb://{}:{}@{}:{}/{}?authSource=admin",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
))
}
Ok(build_mongo_uri(&cfg, true))
}
pub fn build_mongo_uri(cfg: &DatabaseConfig, include_db: bool) -> String {
let is_srv = cfg.port == 0;
let scheme = if is_srv { "mongodb+srv" } else { "mongodb" };
let has_auth = !cfg.username.is_empty() && !cfg.password.is_empty();
let credentials = if has_auth {
format!(
"{}:{}@",
utf8_percent_encode(&cfg.username, USERINFO_ENCODE),
utf8_percent_encode(&cfg.password, USERINFO_ENCODE)
)
} else {
String::new()
};
let authority = if is_srv {
cfg.host.clone()
} else {
format!("{}:{}", cfg.host, cfg.port)
};
let path = if include_db {
format!("/{}", cfg.database)
} else {
"/".to_string()
};
let query = if has_auth { "?authSource=admin" } else { "" };
format!("{}://{}{}{}{}", scheme, credentials, authority, path, query)
}
pub fn extract_db_name(dry_output: &str) -> Option<String> {
let mut dbs = std::collections::HashSet::new();
for line in dry_output.lines() {
if let Some(pos) = line.find("archive prelude ") {
let rest = &line[pos + "archive prelude ".len()..];
if let Some(dot) = rest.find('.') {
let db = &rest[..dot];
dbs.insert(db.to_string());
}
}
}
dbs.into_iter().next()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::services::config::{DatabaseConfig, DbType};
use std::collections::HashMap;
fn cfg(host: &str, port: u16, user: &str, pass: &str) -> DatabaseConfig {
DatabaseConfig {
name: "t".into(),
database: "mydb".into(),
db_type: DbType::MongoDB,
username: user.into(),
password: pass.into(),
port,
host: host.into(),
generated_id: "id".into(),
path: String::new(),
max_packet_size: String::new(),
volume_name: String::new(),
container_name: None,
options: HashMap::new(),
}
}
#[test]
fn standard_with_auth() {
let c = cfg("localhost", 27017, "user", "pass");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb://user:pass@localhost:27017/mydb?authSource=admin"
);
}
#[test]
fn standard_no_auth() {
let c = cfg("localhost", 27017, "", "");
assert_eq!(build_mongo_uri(&c, true), "mongodb://localhost:27017/mydb");
}
#[test]
fn srv_with_auth() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb+srv://user:pass@cluster.example.mongodb.net/mydb?authSource=admin"
);
}
#[test]
fn srv_no_db_for_dryrun() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "pass");
assert_eq!(
build_mongo_uri(&c, false),
"mongodb+srv://user:pass@cluster.example.mongodb.net/?authSource=admin"
);
}
#[test]
fn encodes_special_chars_in_credentials() {
let c = cfg("cluster.example.mongodb.net", 0, "user", "p@ss:w/rd?");
assert_eq!(
build_mongo_uri(&c, true),
"mongodb+srv://user:p%40ss%3Aw%2Frd%3F@cluster.example.mongodb.net/mydb?authSource=admin"
);
}
}
+6 -9
View File
@@ -1,9 +1,11 @@
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -27,21 +29,16 @@ impl Database for MongoDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
)
.await;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
mod backup;
mod restore;
mod connection;
pub mod database;
mod ping;
mod connection;
mod restore;
+6 -2
View File
@@ -19,12 +19,16 @@ pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
Ok(_) => Ok(true),
Err(e) => {
error!("--- MongoDB Connection Error Details ---");
error!("Target Host: {}:{}", cfg.host, cfg.port);
if cfg.port == 0 {
error!("Target Host: {} (srv)", cfg.host);
} else {
error!("Target Host: {}:{}", cfg.host, cfg.port);
}
error!("Error Kind: {:?}", e.kind);
error!("Full Error: {}", e);
error!("Check you database network connectivity");
error!("----------------------------------------");
Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e))
Ok(false)
}
}
}
+47 -9
View File
@@ -1,32 +1,70 @@
use crate::domain::mongodb::connection::{get_mongo_uri, select_mongo_path};
use crate::domain::mongodb::connection::{
build_mongo_uri, extract_db_name, get_mongo_uri, select_mongo_path,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting MongoDB restore for database {}", cfg.name);
logger.log("debug", format!("Starting MongoDB restore for database {}", cfg.name));
let mongorestore = select_mongo_path().join("mongorestore");
let uri = get_mongo_uri(cfg.clone())?;
let output = Command::new(mongorestore)
let dry_start = Instant::now();
let dry_run = Command::new(&mongorestore)
.arg(format!("--uri={}", build_mongo_uri(&cfg, false)))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--dryRun")
.arg("--verbose")
.output()?;
let dry_duration_ms = dry_start.elapsed().as_millis() as f64;
let dry_exit_code = dry_run.status.code().unwrap_or(-1);
let dry_output = String::from_utf8_lossy(&dry_run.stderr);
logger.log_command(
"mongorestore --dryRun",
if dry_output.is_empty() { None } else { Some(dry_output.to_string()) },
Some(dry_exit_code),
Some(dry_duration_ms),
);
let source_db = extract_db_name(&dry_output).unwrap_or_else(|| {
logger.log("info", format!("Could not detect source database from archive, falling back to configured database: {}", cfg.database));
cfg.database.clone()
});
logger.log("info", format!("Using source database in archive: {}", source_db));
let start = Instant::now();
let output = Command::new(&mongorestore)
.arg(format!("--uri={}", uri))
.arg(format!("--archive={}", restore_file.display()))
.arg("--gzip")
.arg("--drop")
.arg(format!("--nsInclude={}.*", source_db))
.arg(format!("--nsFrom={}.*", source_db))
.arg(format!("--nsTo={}.*", cfg.database))
.output()
.with_context(|| format!("Failed to run mongorestore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MongoDB restore failed for {}: {}", cfg.name, stderr);
anyhow::bail!("MongoDB restore failed for : {}", cfg.name);
logger.log_command("mongorestore", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MongoDB restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MongoDB restore failed for: {}", cfg.name);
}
info!("MongoDB restore completed for {}", cfg.name);
logger.log_command("mongorestore", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("MongoDB restore completed for {}", cfg.name));
Ok(())
})
.await?
+56
View File
@@ -0,0 +1,56 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting MSSQL backup for database {}", cfg.name));
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
let connection_string = format!(
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
);
logger.log("info", format!("MSSQL backup: {}:{}/{}{}", cfg.host, cfg.port, cfg.database, file_path.display()));
let start = Instant::now();
let output = Command::new("sqlpackage")
.arg("/a:Export")
.arg(format!("/scs:{}", connection_string))
.arg(format!("/tf:{}", file_path.display()))
.output()
.with_context(|| format!("Failed to run sqlpackage for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
if !output.status.success() {
logger.log("error", format!("MSSQL backup failed — stderr: {} stdout: {}", stderr, stdout));
let out = format!("stderr: {} stdout: {}", stderr, stdout);
logger.log_command("sqlpackage", Some(out), Some(exit_code), Some(duration_ms));
anyhow::bail!("MSSQL backup failed for {}: {}", cfg.name, stderr);
}
let combined = if stdout.is_empty() && stderr.is_empty() {
None
} else {
Some(format!("{}{}", stdout, stderr).trim().to_string())
};
logger.log_command("sqlpackage", combined, Some(0), Some(duration_ms));
logger.log("info", format!("MSSQL backup completed for {}", cfg.name));
Ok(file_path)
})
.await?
}
+18
View File
@@ -0,0 +1,18 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use tiberius::{AuthMethod, Client, Config};
use tokio::net::TcpStream;
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
pub async fn build_client(cfg: &DatabaseConfig) -> Result<Client<Compat<TcpStream>>> {
let mut config = Config::new();
config.host(&cfg.host);
config.port(cfg.port);
config.authentication(AuthMethod::sql_server(&cfg.username, &cfg.password));
config.trust_cert();
let tcp = TcpStream::connect(config.get_addr()).await?;
tcp.set_nodelay(true)?;
let client = Client::connect(config, tcp.compat_write()).await?;
Ok(client)
}
+50
View File
@@ -0,0 +1,50 @@
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MssqlDatabase {
cfg: DatabaseConfig,
}
impl MssqlDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
}
#[async_trait]
impl Database for MssqlDatabase {
fn file_extension(&self) -> &'static str {
".bacpac"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod database;
mod connection;
mod ping;
mod backup;
mod restore;
+24
View File
@@ -0,0 +1,24 @@
use crate::services::config::DatabaseConfig;
use anyhow::Result;
use tracing::{error, info};
pub async fn run(cfg: DatabaseConfig) -> Result<bool> {
info!("Running ping for MSSQL database {}", cfg.name);
match super::connection::build_client(&cfg).await {
Ok(mut client) => match client.simple_query("SELECT 1").await {
Ok(_) => {
info!("MSSQL ping succeeded for {}", cfg.name);
Ok(true)
}
Err(e) => {
error!("MSSQL ping query failed for {}: {:?}", cfg.name, e);
Ok(false)
}
},
Err(e) => {
error!("MSSQL connection failed for {}: {:?}", cfg.name, e);
Ok(false)
}
}
}
+54
View File
@@ -0,0 +1,54 @@
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("debug", format!("Starting MSSQL restore for database {}", cfg.name));
let connection_string = format!(
"Server=tcp:{},{};Database={};User Id={};Password={};TrustServerCertificate=True;Encrypt=False",
cfg.host, cfg.port, cfg.database, cfg.username, cfg.password
);
logger.log("info", format!(
"MSSQL restore: {} → {}:{}/{}",
restore_file.display(),
cfg.host,
cfg.port,
cfg.database
));
let start = Instant::now();
let output = Command::new("sqlpackage")
.arg("/a:Import")
.arg(format!("/tcs:{}", connection_string))
.arg(format!("/sf:{}", restore_file.display()))
.output()
.with_context(|| format!("Failed to run sqlpackage restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
let stdout = String::from_utf8_lossy(&output.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
if !output.status.success() {
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined.clone()) }, Some(exit_code), Some(duration_ms));
logger.log("error", format!(
"MSSQL restore failed for {} — stderr: {} stdout: {}",
cfg.name, stderr, stdout
));
anyhow::bail!("MSSQL restore failed for {}: {}", cfg.name, stderr);
}
logger.log_command("sqlpackage", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("MSSQL restore completed for {}", cfg.name));
Ok(())
})
.await?
}
+36 -20
View File
@@ -1,62 +1,78 @@
use crate::domain::mysql::connection::{server_version};
use crate::domain::mysql::connection::{connection_args, server_version};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
file_extension: &'static str,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
let _version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Mysql version detected: {}", v);
logger.log("debug", format!("MySQL version detected: {}", v));
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
info!("Mysql version found: {}", version);
let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension));
if let Ok(out) = Command::new("mysqldump").arg("--version").output() {
logger.log("debug", format!("mysqldump client: {}", String::from_utf8_lossy(&out.stdout).trim()));
}
logger.log("info", format!("Running mysqldump for {}", cfg.name));
let start = Instant::now();
let output = Command::new("mysqldump")
.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
.args(connection_args(&cfg))
.arg("--routines")
.arg("--events")
.arg("--triggers")
.arg("--verbose")
.arg("--single-transaction")
.arg("--set-gtid-purged=OFF")
.arg("--no-tablespaces")
.arg("--quick")
.arg("--add-drop-database")
.arg("--databases")
.arg(cfg.database)
.arg("-r")
.arg(&file_path)
.arg("--skip-lock-tables")
.arg("--skip-add-drop-table")
.arg("--no-create-db")
.arg("--default-character-set=utf8mb4")
.arg("--network-timeout")
.arg(format!("--max-allowed-packet={}", cfg.max_packet_size))
.arg(&cfg.database)
.arg("-r").arg(&file_path)
.envs(env)
.output()
.with_context(|| format!("Failed to run mysqldump for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let _stdout = String::from_utf8_lossy(&output.stdout).to_string();
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
logger.log_command("mysqldump", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
anyhow::bail!("MySQL backup failed for {}: {}", cfg.name, stderr);
}
logger.log_command("mysqldump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("mysqldump completed for {}", cfg.name));
Ok(file_path)
})
.await?
+29 -8
View File
@@ -1,14 +1,36 @@
use crate::services::config::DatabaseConfig;
use std::process::Command;
use anyhow::Result;
use std::process::Command;
pub fn connection_args(cfg: &DatabaseConfig) -> Vec<String> {
let protocol = cfg
.options
.get("protocol")
.and_then(|v| v.as_str())
.unwrap_or("tcp");
let mut args = vec![
format!("--protocol={}", protocol),
"--host".to_string(),
cfg.host.clone(),
"--port".to_string(),
cfg.port.to_string(),
"--user".to_string(),
cfg.username.clone(),
];
if let Some(socket) = cfg.options.get("socket").and_then(|v| v.as_str()) {
args.push(format!("--socket={}", socket));
}
args
}
pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let output = Command::new("mysql")
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--user").arg(&cfg.username)
.arg("-e").arg("SELECT VERSION();")
.args(connection_args(cfg))
.arg("-e")
.arg("SELECT VERSION();")
.env("MYSQL_PWD", &cfg.password)
.output()?;
@@ -19,11 +41,10 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
let version = String::from_utf8_lossy(&output.stdout)
.lines()
.nth(1) // skip column header
.nth(1)
.unwrap_or_default()
.trim()
.to_string();
Ok(version)
}
+18 -12
View File
@@ -1,14 +1,13 @@
use std::collections::HashMap;
use anyhow::Result;
use async_trait::async_trait;
use std::path::{Path, PathBuf};
use super::{
backup,
ping, restore,
};
use super::{backup, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct MySQLDatabase {
cfg: DatabaseConfig,
@@ -36,16 +35,23 @@ impl Database for MySQLDatabase {
ping::run(self.cfg.clone(), self.build_env().clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await;
let res = backup::run(
self.cfg.clone(),
dir.to_path_buf(),
self.build_env().clone(),
self.file_extension(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf()).await;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
pub mod backup;
pub mod connection;
pub mod database;
mod restore;
mod ping;
mod connection;
mod restore;
+3 -8
View File
@@ -1,17 +1,12 @@
use crate::domain::mysql::connection::connection_args;
use crate::services::config::DatabaseConfig;
use std::collections::HashMap;
use tokio::process::Command;
use tokio::time::{Duration, timeout};
pub async fn run(cfg: DatabaseConfig, env: HashMap<String, String>) -> anyhow::Result<bool> {
let mut cmd = Command::new("mysqladmin");
cmd.arg("--host")
.arg(cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(cfg.username)
let mut cmd = Command::new("mariadb-admin");
cmd.args(connection_args(&cfg))
.arg("ping")
.envs(env);
+34 -32
View File
@@ -1,52 +1,51 @@
use crate::services::backup::logger::JobLogger;
use crate::domain::mysql::connection::connection_args;
use crate::services::config::DatabaseConfig;
use anyhow::{Context, Result};
use std::fs::File;
use std::io::{Read, Write};
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf, logger: Arc<JobLogger>) -> Result<()> {
let handle = tokio::task::spawn_blocking(move || -> Result<()> {
debug!("Starting restore for database {}", cfg.name);
logger.log("info", format!("Starting restore for database {}", cfg.name));
let mut sql_content = String::new();
let mut file = File::open(&restore_file)
.with_context(|| format!("Failed to open restore file {}", restore_file.display()))?;
file.read_to_string(&mut sql_content)
.with_context(|| format!("Failed to read restore file {}", restore_file.display()))?;
let drop_create_cmd = format!(
"DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};",
"DROP DATABASE IF EXISTS `{0}`; CREATE DATABASE `{0}`;",
cfg.database
);
let drop_status = Command::new("mysql")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
let drop_start = Instant::now();
let drop_output = Command::new("mysql")
.args(connection_args(&cfg))
.arg("-e")
.arg(&drop_create_cmd)
.env("MYSQL_PWD", &cfg.password)
.status()
.output()
.with_context(|| format!("Failed to drop/recreate database {}", cfg.name))?;
if !drop_status.success() {
error!("Drop/create database failed for {}", cfg.name);
let drop_duration_ms = drop_start.elapsed().as_millis() as f64;
let drop_exit_code = drop_output.status.code().unwrap_or(-1);
let drop_stderr = String::from_utf8_lossy(&drop_output.stderr).to_string();
if !drop_output.status.success() {
logger.log_command("mysql", Some(drop_stderr.clone()), Some(drop_exit_code), Some(drop_duration_ms));
logger.log("error", format!("Drop/create database failed for {}: {}", cfg.name, drop_stderr));
anyhow::bail!("Failed to drop/recreate database {}", cfg.name);
}
info!("Database {} dropped and recreated", cfg.name);
logger.log_command("mysql", if drop_stderr.is_empty() { None } else { Some(drop_stderr) }, Some(0), Some(drop_duration_ms));
logger.log("info", format!("Database {} dropped and recreated", cfg.name));
let start = Instant::now();
let mut child = Command::new("mysql")
.arg("--host")
.arg(&cfg.host)
.arg("--port")
.arg(cfg.port.to_string())
.arg("--user")
.arg(&cfg.username)
.args(connection_args(&cfg))
.arg("--database")
.arg(&cfg.database)
.env("MYSQL_PWD", &cfg.password)
.stdin(std::process::Stdio::piped())
@@ -54,23 +53,26 @@ pub async fn run(cfg: DatabaseConfig, restore_file: PathBuf) -> Result<()> {
.with_context(|| format!("Failed to start mysql restore for {}", cfg.name))?;
let mut stdin = child.stdin.take().context("Failed to open child stdin")?;
stdin
.write_all(sql_content.as_bytes())
.context("Failed to write SQL content to mysql stdin")?;
stdin.flush()?;
std::io::copy(&mut file, &mut stdin)
.context("Failed to stream SQL content to mysql stdin")?;
drop(stdin);
let output = child
.wait_with_output()
.with_context(|| format!("Failed to complete mysql restore for {}", cfg.name))?;
let duration_ms = start.elapsed().as_millis() as f64;
let exit_code = output.status.code().unwrap_or(-1);
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
error!("MySQL restore failed for {}: {}", cfg.name, stderr);
logger.log_command("mysql", Some(stderr.clone()), Some(exit_code), Some(duration_ms));
logger.log("error", format!("MySQL restore failed for {}: {}", cfg.name, stderr));
anyhow::bail!("MySQL restore failed for {}", cfg.name);
}
info!("Restore finished successfully for database {}", cfg.name);
logger.log_command("mysql", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Restore finished successfully for database {}", cfg.name));
Ok(())
});
+74 -65
View File
@@ -1,141 +1,150 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use tracing::{debug, error, info};
use std::sync::Arc;
use std::time::Instant;
use super::connection::{select_pg_path, server_version};
use super::format::PostgresDumpFormat;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
format: PostgresDumpFormat,
backup_dir: PathBuf,
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
debug!("Starting backup for database {}", cfg.name);
logger.log("info", format!("Starting backup for database {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => {
debug!("Postgres version detected: {}", v);
logger.log("debug", format!("Postgres version detected: {}", v));
v
}
Err(e) => {
error!("Failed to get server version for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to get server version: {}", e));
return Err(e.into());
}
};
let pg_dump = select_pg_path(&version).join("pg_dump");
debug!("Using pg_dump at {:?}", pg_dump);
logger.log("debug", format!("Using pg_dump at {:?}", pg_dump));
match format {
PostgresDumpFormat::Fc => {
info!("Running FC backup for {}", cfg.name);
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
);
logger.log("info", format!("Running FC backup for {}", cfg.name));
let status = Command::new(&pg_dump)
.arg("--dbname")
.arg(&url)
let file_path = backup_dir.join(format!("{}.dump", cfg.generated_id));
let start = Instant::now();
let output = Command::new(&pg_dump)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-Fc")
.arg("-f")
.arg(&file_path)
.arg("-f").arg(&file_path)
.arg("-v")
.arg("--compress=3")
.status();
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match status {
Ok(s) if s.success() => info!(
"FC backup completed successfully for {} at {:?}",
cfg.name, file_path
),
Ok(s) => {
error!("FC backup failed with status {:?} for {}", s, cfg.name);
anyhow::bail!("Postgres backup failed for {}", cfg.name);
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log("info", format!("FC backup completed successfully for {} at {:?}", cfg.name, file_path));
logger.log_command("pg_dump", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
} else {
logger.log("error", format!("FC backup failed with status {:?} for {}", o.status, cfg.name));
logger.log_command("pg_dump", Some(stderr), Some(exit_code), Some(duration_ms));
anyhow::bail!("Postgres backup failed for {}", cfg.name);
}
}
Err(e) => {
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
logger.log_command("pg_dump", Some(e.to_string()), Some(-1), Some(duration_ms));
return Err(e.into());
}
}
info!("Backup finished for database {}", cfg.name);
logger.log("info", format!("Backup finished for database {}", cfg.name));
Ok(file_path)
}
PostgresDumpFormat::Fd => {
info!("Running FD backup for {}", cfg.name);
logger.log("info", format!("Running FD backup for {}", cfg.name));
let dump_dir = backup_dir.join(format!("{}_dir", cfg.generated_id));
let tar_file = backup_dir.join(format!("{}.tar.gz", cfg.generated_id));
if let Err(e) = std::fs::create_dir_all(&dump_dir) {
error!(
"Failed to create dump directory {:?} for {}: {:?}",
dump_dir, cfg.name, e
);
logger.log("error", format!("Failed to create dump directory {:?} for {}: {:?}", dump_dir, cfg.name, e));
return Err(e.into());
}
let url = format!(
"postgresql://{}:{}@{}:{}/{}",
cfg.username, cfg.password, cfg.host, cfg.port, cfg.database
);
let cmd_label = format!("pg_dump -Fd {}@{}:{}/{}", cfg.username, cfg.host, cfg.port, cfg.database);
let status = Command::new(&pg_dump)
.arg("--dbname")
.arg(&url)
let start = Instant::now();
let output = Command::new(&pg_dump)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg(&cfg.database)
.arg("-Fd")
.arg("-j")
.arg("4")
.arg("-f")
.arg(&dump_dir)
.arg("-j").arg("4")
.arg("-f").arg(&dump_dir)
.arg("-v")
.status();
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match status {
Ok(s) if s.success() => {
info!("FD backup pg_dump completed successfully for {}", cfg.name)
}
Ok(s) => {
error!(
"FD backup pg_dump failed with status {:?} for {}",
s, cfg.name
);
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log("info", format!("FD backup pg_dump completed successfully for {}", cfg.name));
logger.log_command(cmd_label, if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
} else {
logger.log("error", format!("FD backup pg_dump failed with status {:?} for {}", o.status, cfg.name));
logger.log_command(cmd_label, Some(stderr), Some(exit_code), Some(duration_ms));
anyhow::bail!("Postgres FD backup failed for {}", cfg.name);
}
}
Err(e) => {
error!("Error executing pg_dump for {}: {:?}", cfg.name, e);
logger.log("error", format!("Error executing pg_dump for {}: {:?}", cfg.name, e));
logger.log_command(cmd_label, Some(e.to_string()), Some(-1), Some(duration_ms));
return Err(e.into());
}
}
match std::fs::File::create(&tar_file) {
Ok(tar_gz) => {
let enc =
flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
let enc = flate2::write::GzEncoder::new(tar_gz, flate2::Compression::default());
let mut tar = tar::Builder::new(enc);
if let Err(e) = tar.append_dir_all(".", &dump_dir) {
error!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to append dump_dir to tar for {}: {:?}", cfg.name, e));
return Err(e.into());
}
if let Err(e) = tar.finish() {
error!("Failed to finish tar archive for {}: {:?}", cfg.name, e);
logger.log("error", format!("Failed to finish tar archive for {}: {:?}", cfg.name, e));
return Err(e.into());
}
info!("FD backup archive created at {:?}", tar_file);
logger.log("info", format!("FD backup archive created at {:?}", tar_file));
}
Err(e) => {
error!(
"Failed to create tar.gz file {:?} for {}: {:?}",
tar_file, cfg.name, e
);
logger.log("error", format!("Failed to create tar.gz file {:?} for {}: {:?}", tar_file, cfg.name, e));
return Err(e.into());
}
}
info!("Backup finished for database {}", cfg.name);
logger.log("info", format!("Backup finished for database {}", cfg.name));
Ok(tar_file)
}
}
+25
View File
@@ -0,0 +1,25 @@
use crate::services::config::DatabaseConfig;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RestoreCleanMode {
None,
Clean,
DropSchemas,
DropDatabase,
}
impl RestoreCleanMode {
pub fn from_config(cfg: &DatabaseConfig) -> (Self, Option<String>) {
match cfg.options.get("clean_mode").and_then(|v| v.as_str()) {
None | Some("clean") => (Self::Clean, None),
Some("none") => (Self::None, None),
Some("drop_schemas") => (Self::DropSchemas, None),
Some("drop_database") => (Self::DropDatabase, None),
Some(other) => (Self::Clean, Some(other.to_string())),
}
}
pub fn uses_pg_restore_clean(self) -> bool {
matches!(self, Self::Clean)
}
}
+81
View File
@@ -0,0 +1,81 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
use super::super::connection::{
is_superuser, pg_dumpall_binary_name, select_pg_path, server_version,
};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
backup_dir: PathBuf,
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<PathBuf> {
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
logger.log("info", format!("Starting cluster backup for {}", cfg.name));
let version = match futures::executor::block_on(server_version(&cfg)) {
Ok(v) => v,
Err(e) => {
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
return Err(e.into());
}
};
match futures::executor::block_on(is_superuser(&cfg)) {
Ok(true) => {}
Ok(false) => {
logger.log("error", format!("postgresql-cluster backup requires a superuser role for {}", cfg.name));
anyhow::bail!("postgresql-cluster backup requires a superuser role for {}", cfg.name);
}
Err(e) => {
logger.log("error", format!("Failed to check superuser status for {}: {:?}", cfg.name, e));
return Err(e.into());
}
}
let pg_dumpall = select_pg_path(&version).join(pg_dumpall_binary_name());
let file_path = backup_dir.join(format!("{}.sql", cfg.generated_id));
logger.log("info", format!("Running pg_dumpall for cluster {} via {:?}", cfg.name, pg_dumpall));
let start = Instant::now();
let output = Command::new(&pg_dumpall)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--clean")
.arg("--if-exists")
.arg("-v")
.arg("-f").arg(&file_path)
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log_command("pg_dumpall", if stderr.is_empty() { None } else { Some(stderr) }, Some(0), Some(duration_ms));
logger.log("info", format!("Cluster backup completed for {} at {:?}", cfg.name, file_path));
Ok(file_path)
} else {
logger.log_command("pg_dumpall", Some(stderr), Some(exit_code), Some(duration_ms));
anyhow::bail!("Cluster backup (pg_dumpall) failed for {}", cfg.name);
}
}
Err(e) => {
logger.log_command("pg_dumpall", Some(e.to_string()), Some(-1), Some(duration_ms));
Err(e.into())
}
}
})
.await?
}
+53
View File
@@ -0,0 +1,53 @@
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::super::ping;
use super::{backup, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
pub struct PostgresClusterDatabase {
pub cfg: DatabaseConfig,
}
impl PostgresClusterDatabase {
pub fn new(cfg: DatabaseConfig) -> Self {
Self { cfg }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
impl Database for PostgresClusterDatabase {
fn file_extension(&self) -> &'static str {
".sql"
}
async fn ping(&self) -> Result<bool> {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), file.to_path_buf(), self.build_env(), logger).await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
}
+3
View File
@@ -0,0 +1,3 @@
pub mod backup;
pub mod database;
pub mod restore;
+85
View File
@@ -0,0 +1,85 @@
use anyhow::Result;
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use std::sync::Arc;
use std::time::Instant;
use super::super::connection::{is_superuser, psql_binary_name, select_pg_path, server_version, terminate_all_connections};
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
pub async fn run(
cfg: DatabaseConfig,
restore_file: PathBuf,
env: HashMap<String, String>,
logger: Arc<JobLogger>,
) -> Result<()> {
let handle = tokio::runtime::Handle::current();
tokio::task::spawn_blocking(move || -> Result<()> {
logger.log("info", format!("Starting cluster restore for {}", cfg.name));
let version = match handle.block_on(server_version(&cfg)) {
Ok(v) => v,
Err(e) => {
logger.log("error", format!("Failed to get server version for {}: {:?}", cfg.name, e));
return Err(e.into());
}
};
match handle.block_on(is_superuser(&cfg)) {
Ok(true) => {}
Ok(false) => {
logger.log("error", format!("postgresql-cluster restore requires a superuser role for {}", cfg.name));
anyhow::bail!("postgresql-cluster restore requires a superuser role for {}", cfg.name);
}
Err(e) => {
logger.log("error", format!("Failed to check superuser status for {}: {:?}", cfg.name, e));
return Err(e.into());
}
}
let psql = select_pg_path(&version).join(psql_binary_name());
if let Err(e) = handle.block_on(terminate_all_connections(&cfg)) {
logger.log("error", format!("Failed to terminate connections for cluster {}: {:?}", cfg.name, e));
return Err(e.into());
}
logger.log("info", format!("All user database connections terminated for cluster {}", cfg.name));
logger.log("info", format!("Replaying cluster dump for {} via {:?}", cfg.name, psql));
let start = Instant::now();
let output = Command::new(&psql)
.arg("--host").arg(&cfg.host)
.arg("--port").arg(cfg.port.to_string())
.arg("--username").arg(&cfg.username)
.arg("--dbname").arg("postgres")
.arg("-f").arg(&restore_file)
.envs(env)
.output();
let duration_ms = start.elapsed().as_millis() as f64;
match output {
Ok(o) => {
let stderr = String::from_utf8_lossy(&o.stderr).to_string();
let stdout = String::from_utf8_lossy(&o.stdout).to_string();
let combined = format!("{}{}", stdout, stderr);
let exit_code = o.status.code().unwrap_or(-1);
if o.status.success() {
logger.log_command("psql", if combined.is_empty() { None } else { Some(combined) }, Some(0), Some(duration_ms));
logger.log("info", format!("Cluster restore completed for {}", cfg.name));
Ok(())
} else {
logger.log_command("psql", if combined.is_empty() { None } else { Some(combined) }, Some(exit_code), Some(duration_ms));
anyhow::bail!("Cluster restore (psql) failed for {}", cfg.name);
}
}
Err(e) => {
logger.log_command("psql", Some(e.to_string()), Some(-1), Some(duration_ms));
Err(e.into())
}
}
})
.await?
}
+294 -6
View File
@@ -1,18 +1,23 @@
use crate::domain::postgres::format::PostgresDumpFormat;
use crate::services::config::DatabaseConfig;
use crate::settings::CONFIG;
use anyhow::Result;
use std::path::Path;
use tokio_postgres::{Client, NoTls};
use tokio_postgres::{Client, Config, NoTls};
use tracing::{error, info};
pub async fn connect(cfg: &DatabaseConfig) -> Result<Client> {
info!("Connecting to postgres database {}:{}", cfg.host, cfg.port);
let dsn = format!(
"host={} port={} user={} password={} dbname={}",
cfg.host, cfg.port, cfg.username, cfg.password, cfg.database
);
let (client, connection) = tokio_postgres::connect(&dsn, NoTls).await?;
let mut config = Config::new();
config
.host(&cfg.host)
.port(cfg.port)
.user(&cfg.username)
.password(&cfg.password)
.dbname(&cfg.database);
let (client, connection) = config.connect(NoTls).await?;
tokio::spawn(async move {
if let Err(e) = connection.await {
error!("Postgres connection error: {}", e);
@@ -28,11 +33,133 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result<String> {
Ok(version)
}
pub async fn server_version_major(cfg: &DatabaseConfig) -> Result<u32> {
let v = server_version(cfg).await?;
Ok(v.split(['.', ' '])
.next()
.and_then(|s| s.parse().ok())
.unwrap_or(17))
}
pub async fn is_superuser(cfg: &DatabaseConfig) -> Result<bool> {
let client = connect(cfg).await?;
let is_super: bool = client
.query_one("SELECT current_setting('is_superuser') = 'on';", &[])
.await?
.get(0);
Ok(is_super)
}
pub async fn can_drop_database(cfg: &DatabaseConfig) -> Result<bool> {
let client = connect(cfg).await?;
let row = client
.query_one(
"SELECT r.rolsuper OR (r.rolcreatedb AND pg_catalog.pg_has_role(current_user, d.datdba, 'USAGE')) \
FROM pg_roles r, pg_database d \
WHERE r.rolname = current_user AND d.datname = current_database()",
&[],
)
.await?;
Ok(row.get(0))
}
pub fn select_pg_path(version: &str) -> std::path::PathBuf {
select_pg_path_with(version, &CONFIG.pg_bin_dir)
}
pub(crate) fn select_pg_path_with(version: &str, pg_bin_dir: &str) -> std::path::PathBuf {
let major = version.split('.').next().unwrap_or("17");
if !pg_bin_dir.is_empty() {
return pg_bin_dir.into();
}
let candidates: Vec<std::path::PathBuf> = if cfg!(target_os = "windows") {
vec![
// Default install path used by the official EDB Windows installer
format!(r"C:\Program Files\PostgreSQL\{major}\bin").into(),
format!(r"C:\Program Files (x86)\PostgreSQL\{major}\bin").into(),
]
} else if cfg!(target_os = "macos") {
vec![
// Homebrew on Apple Silicon
format!("/opt/homebrew/opt/postgresql@{major}/bin").into(),
// Homebrew on Intel
format!("/usr/local/opt/postgresql@{major}/bin").into(),
// Postgres.app
format!("/Applications/Postgres.app/Contents/Versions/{major}/bin").into(),
]
} else {
vec![
// Debian/Ubuntu packages
format!("/usr/lib/postgresql/{major}/bin").into(),
// Common RPM-based distro layout
format!("/usr/pgsql-{major}/bin").into(),
]
};
if let Some(found) = candidates.into_iter().find(|p| pg_dump_exists_in(p)) {
return found;
}
if let Some(dir) = find_pg_dump_in_path() {
return dir;
}
format!("/usr/lib/postgresql/{}/bin", major).into()
}
pub(crate) fn pg_dump_binary_name() -> &'static str {
if cfg!(target_os = "windows") {
"pg_dump.exe"
} else {
"pg_dump"
}
}
pub(crate) fn pg_dumpall_binary_name() -> &'static str {
if cfg!(target_os = "windows") {
"pg_dumpall.exe"
} else {
"pg_dumpall"
}
}
pub(crate) fn psql_binary_name() -> &'static str {
if cfg!(target_os = "windows") {
"psql.exe"
} else {
"psql"
}
}
pub(crate) fn pg_restore_binary_name() -> &'static str {
if cfg!(target_os = "windows") {
"pg_restore.exe"
} else {
"pg_restore"
}
}
pub(crate) fn quote_ident(s: &str) -> String {
format!("\"{}\"", s.replace('"', "\"\""))
}
pub(crate) fn quote_literal(s: &str) -> String {
format!("'{}'", s.replace('\'', "''"))
}
pub(crate) fn pg_dump_exists_in(dir: &std::path::Path) -> bool {
dir.join(pg_dump_binary_name()).is_file()
}
fn find_pg_dump_in_path() -> Option<std::path::PathBuf> {
let path_var = std::env::var_os("PATH")?;
std::env::split_paths(&path_var).find(|dir| pg_dump_exists_in(dir))
}
pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
let mut admin = cfg.clone();
admin.database = "postgres".to_string().into();
@@ -54,6 +181,128 @@ pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> {
Ok(())
}
pub async fn terminate_all_connections(cfg: &DatabaseConfig) -> Result<()> {
let mut admin = cfg.clone();
admin.database = "postgres".to_string().into();
let client = connect(&admin).await?;
client
.execute(
r#"
SELECT pg_terminate_backend(pid)
FROM pg_stat_activity
WHERE datname NOT IN ('postgres', 'template0', 'template1')
AND pid <> pg_backend_pid();
"#,
&[],
)
.await?;
Ok(())
}
pub async fn drop_and_recreate_database(cfg: &DatabaseConfig) -> Result<()> {
let mut admin_cfg = cfg.clone();
admin_cfg.database = "postgres".to_string();
let admin = connect(&admin_cfg).await?;
let row = admin
.query_opt(
r#"
SELECT pg_encoding_to_char(encoding), datcollate, datctype,
pg_get_userbyid(datdba), datistemplate
FROM pg_database WHERE datname = $1
"#,
&[&cfg.database],
)
.await?;
let (encoding, collate, ctype, owner) = match &row {
Some(r) => (
r.get::<_, String>(0),
r.get::<_, String>(1),
r.get::<_, String>(2),
r.get::<_, String>(3),
),
None => ("UTF8".into(), "C".into(), "C".into(), cfg.username.clone()),
};
if let Some(r) = &row {
if r.get::<_, bool>(4) {
anyhow::bail!("Refusing to drop template database {}", cfg.database);
}
}
let db = quote_ident(&cfg.database);
if let Err(e) = admin
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS false"))
.await
{
tracing::warn!("ALLOW_CONNECTIONS false failed for {}: {e}", cfg.database);
}
let major = server_version_major(&admin_cfg).await?;
let drop_stmt = if major >= 13 {
format!("DROP DATABASE IF EXISTS {db} WITH (FORCE)")
} else {
format!("DROP DATABASE IF EXISTS {db}")
};
let mut last_err = None;
let mut dropped = false;
for _ in 0..3 {
let _ = terminate_connections(cfg).await;
match admin.batch_execute(&drop_stmt).await {
Ok(()) => {
dropped = true;
break;
}
Err(e) => {
last_err = Some(e);
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
}
}
if !dropped {
let _ = admin
.batch_execute(&format!("ALTER DATABASE {db} WITH ALLOW_CONNECTIONS true"))
.await;
return Err(last_err
.map(anyhow::Error::from)
.unwrap_or_else(|| anyhow::anyhow!("DROP DATABASE {} failed", cfg.database)));
}
admin
.batch_execute(&format!(
"CREATE DATABASE {db} OWNER {} TEMPLATE template0 ENCODING {} LC_COLLATE {} LC_CTYPE {}",
quote_ident(&owner),
quote_literal(&encoding),
quote_literal(&collate),
quote_literal(&ctype),
))
.await?;
Ok(())
}
pub fn sniff_format(restore_file: &Path) -> Result<PostgresDumpFormat> {
use std::io::Read;
let mut f = std::fs::File::open(restore_file)?;
let mut magic = [0u8; 5];
let n = f.read(&mut magic)?;
let head = &magic[..n];
if head.starts_with(b"PGDMP") {
Ok(PostgresDumpFormat::Fc)
} else if head.starts_with(&[0x1f, 0x8b]) {
Ok(PostgresDumpFormat::Fd)
} else {
anyhow::bail!("Unrecognized dump format for {:?}", restore_file)
}
}
pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
match restore_file.extension().and_then(|e| e.to_str()) {
Some("dump") => PostgresDumpFormat::Fc,
@@ -63,11 +312,50 @@ pub fn detect_format_from_file(restore_file: &Path) -> PostgresDumpFormat {
}
}
pub async fn drop_all_schemas(cfg: &DatabaseConfig) -> Result<Vec<String>> {
let client = connect(cfg).await?;
let rows = client
.query(
r#"
SELECT nspname FROM pg_namespace
WHERE nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
AND nspname NOT LIKE 'pg\_temp\_%'
AND nspname NOT LIKE 'pg\_toast\_temp\_%'
ORDER BY nspname
"#,
&[],
)
.await?;
let schemas: Vec<String> = rows.iter().map(|r| r.get::<_, String>(0)).collect();
for s in &schemas {
client
.batch_execute(&format!("DROP SCHEMA IF EXISTS {} CASCADE", quote_ident(s)))
.await?;
}
client
.batch_execute("SELECT lo_unlink(oid) FROM pg_largeobject_metadata")
.await
.ok();
Ok(schemas)
}
pub async fn recreate_public_schema(cfg: &DatabaseConfig, owner: &str) -> Result<()> {
let client = connect(cfg).await?;
client
.batch_execute(&format!(
"CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION {}; GRANT USAGE ON SCHEMA public TO PUBLIC;",
quote_ident(owner)
))
.await?;
Ok(())
}
pub async fn detect_format_from_size(cfg: &DatabaseConfig) -> PostgresDumpFormat {
info!(
"Detecting database format {:?} - {:?}",
cfg.name, cfg.generated_id
);
let client = match connect(cfg).await {
Ok(c) => c,
Err(_) => return PostgresDumpFormat::Fc,
+28 -10
View File
@@ -1,13 +1,11 @@
use anyhow::Result;
use async_trait::async_trait;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use super::{
backup,
format::PostgresDumpFormat,
ping, restore,
};
use std::sync::Arc;
use super::{backup, format::PostgresDumpFormat, ping, restore};
use crate::domain::factory::Database;
use crate::services::backup::logger::JobLogger;
use crate::services::config::DatabaseConfig;
use crate::utils::locks::{DbOpLock, FileLock};
@@ -20,6 +18,12 @@ impl PostgresDatabase {
pub fn new(cfg: DatabaseConfig, format: PostgresDumpFormat) -> Self {
Self { cfg, format }
}
fn build_env(&self) -> HashMap<String, String> {
let mut envs = std::env::vars().collect::<HashMap<_, _>>();
envs.insert("PGPASSWORD".to_string(), self.cfg.password.to_string());
envs
}
}
#[async_trait]
@@ -35,16 +39,30 @@ impl Database for PostgresDatabase {
ping::run(self.cfg.clone()).await
}
async fn backup(&self, dir: &Path) -> Result<PathBuf> {
async fn backup(&self, dir: &Path, logger: Arc<JobLogger>) -> Result<PathBuf> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?;
let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await;
let res = backup::run(
self.cfg.clone(),
self.format,
dir.to_path_buf(),
self.build_env(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
async fn restore(&self, file: &Path) -> Result<()> {
async fn restore(&self, file: &Path, logger: Arc<JobLogger>) -> Result<()> {
FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?;
let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await;
let res = restore::run(
self.cfg.clone(),
self.format,
file.to_path_buf(),
self.build_env(),
logger,
)
.await;
FileLock::release(&self.cfg.generated_id).await?;
res
}
+2 -2
View File
@@ -1,5 +1,5 @@
#[derive(Clone, Copy)]
#[derive(Clone, Copy, PartialEq, Debug)]
pub enum PostgresDumpFormat {
Fc,
Fd,
}
}

Some files were not shown because too many files have changed in this diff Show More