diff -Nru pgbackrest-2.36/debian/changelog pgbackrest-2.37/debian/changelog --- pgbackrest-2.36/debian/changelog 2021-12-03 13:34:51.000000000 +0000 +++ pgbackrest-2.37/debian/changelog 2022-01-04 10:44:12.000000000 +0000 @@ -1,8 +1,28 @@ -pgbackrest (2.36-1build1) jammy; urgency=medium +pgbackrest (2.37-1) unstable; urgency=medium - * No-change rebuild against openssl3 + * New Upstream Release: + - Bug Fixes: + * Fix restore delta link mapping when path/file already exists. + * Fix socket leak on connection retries. + - Features: + * Add TLS server. + * Add --cmd option. + - Improvements: + * Check archive immediately after backup start. + * Add timeline and checkpoint checks to backup. + * Check that clusters are alive and correctly configured during a backup. + * Error when restore is unable to find a backup to match the time target. + * Parse protocol/port in S3/Azure endpoints. + * Add warning when checkpoint_timeout exceeds db-timeout. + * Add verb to HTTP error output. + * Allow y/n arguments for boolean command-line options. + * Make backup size logging exactly match info command output. + - Documentation Improvements: + * Display size option default and allowed values with appropriate units. + * Fix typos and improve documentation for the tablespace-map-all option. + * Remove obsolete statement about future multi-repository support. - -- Simon Chopin Fri, 03 Dec 2021 14:34:51 +0100 + -- Adrian Vondendriesch Tue, 04 Jan 2022 11:44:12 +0100 pgbackrest (2.36-1) unstable; urgency=medium diff -Nru pgbackrest-2.36/doc/RELEASE.md pgbackrest-2.37/doc/RELEASE.md --- pgbackrest-2.36/doc/RELEASE.md 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/RELEASE.md 2022-01-03 13:43:55.000000000 +0000 @@ -190,7 +190,7 @@ These scripts are required by `src/config` and should be updated after each release, when needed. Note that these files are updated very infrequently. -Check the latest version of `automake` and see if it is > `1.16.4`: +Check the latest version of `automake` and see if it is > `1.16.5`: ``` https://git.savannah.gnu.org/gitweb/?p=automake.git ``` diff -Nru pgbackrest-2.36/doc/resource/exe.cache pgbackrest-2.37/doc/resource/exe.cache --- pgbackrest-2.36/doc/resource/exe.cache 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/resource/exe.cache 2022-01-03 13:43:55.000000000 +0000 @@ -335,7 +335,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo mkdir -p /build/pgbackrest-release-2.36" + "sudo mkdir -p /build/pgbackrest-release-2.37" ], "host" : "build", "load-env" : true, @@ -348,7 +348,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo cp -r /pgbackrest/src /build/pgbackrest-release-2.36" + "sudo cp -r /pgbackrest/src /build/pgbackrest-release-2.37" ], "host" : "build", "load-env" : true, @@ -361,7 +361,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo chown -R vagrant /build/pgbackrest-release-2.36" + "sudo chown -R vagrant /build/pgbackrest-release-2.37" ], "host" : "build", "load-env" : true, @@ -402,7 +402,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "cd /build/pgbackrest-release-2.36/src && ./configure && make" + "cd /build/pgbackrest-release-2.37/src && ./configure && make" ], "cmd-extra" : "-j 4", "host" : "build", @@ -444,7 +444,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo scp build:/build/pgbackrest-release-2.36/src/pgbackrest /usr/bin" + "sudo scp build:/build/pgbackrest-release-2.37/src/pgbackrest /usr/bin" ], "cmd-extra" : "2>&1", "host" : "pg-primary", @@ -572,7 +572,7 @@ "type" : "exe", "value" : { "output" : [ - "pgBackRest 2.36 - General help", + "pgBackRest 2.37 - General help", "", "Usage:", " pgbackrest [options] [command]", @@ -588,6 +588,8 @@ " repo-get Get a file from a repository.", " repo-ls List files in a repository.", " restore Restore a database cluster.", + " server pgBackRest server.", + " server-ping Ping pgBackRest server.", " stanza-create Create the required stanza data.", " stanza-delete Delete a stanza.", " stanza-upgrade Upgrade a stanza.", @@ -732,7 +734,7 @@ "type" : "exe", "value" : { "output" : [ - "pgBackRest 2.36 - 'backup' command - 'log-path' option help", + "pgBackRest 2.37 - 'backup' command - 'log-path' option help", "", "Path where log files are stored.", "", @@ -969,7 +971,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stanza-create command begin 2.36: --exec-id=1366-f2772c57 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", + "P00 INFO: stanza-create command begin 2.37: --exec-id=1293-896bc18e --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", "P00 INFO: stanza-create for stanza 'demo' on repo1", "P00 INFO: stanza-create command end: completed successfully" ] @@ -996,10 +998,10 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: check command begin 2.36: --exec-id=1377-3cf7b456 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", + "P00 INFO: check command begin 2.37: --exec-id=1302-851f6789 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", "P00 INFO: check repo1 configuration (primary)", "P00 INFO: check repo1 archive for WAL (primary)", - "P00 INFO: WAL segment 000000010000000000000001 successfully archived to '/var/lib/pgbackrest/archive/demo/12-1/0000000100000000/000000010000000000000001-121842c5b2b35528b72ba937561b957d77d6b7b1.gz' on repo1", + "P00 INFO: WAL segment 000000010000000000000001 successfully archived to '/var/lib/pgbackrest/archive/demo/12-1/0000000100000000/000000010000000000000001-573dbc22811c4f0055caa67a7cf20c9b1d4a5a35.gz' on repo1", "P00 INFO: check command end: completed successfully" ] } @@ -1056,16 +1058,16 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: backup command begin 2.36: --exec-id=1408-368138d0 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo --start-fast", + "P00 INFO: backup command begin 2.37: --exec-id=1330-32ed25fb --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo --start-fast", "P00 WARN: no prior backup exists, incr backup has been changed to full", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", "P00 INFO: backup start archive = 000000010000000000000002, lsn = 0/2000028", - " [filtered 2 lines of output]", - "P00 INFO: check archive for segment(s) 000000010000000000000002:000000010000000000000002", - "P00 INFO: new backup label = 20211029-120551F", + " [filtered 3 lines of output]", + "P00 INFO: check archive for segment(s) 000000010000000000000002:000000010000000000000003", + "P00 INFO: new backup label = 20211231-195532F", "P00 INFO: full backup size = 23.4MB, file total = 976", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1408-368138d0 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=1330-32ed25fb --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" ] } }, @@ -1083,7 +1085,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120551F" + "20211231-195532F" ] } }, @@ -1109,12 +1111,12 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 6 lines of output]", - "P00 INFO: check archive for segment(s) 000000010000000000000003:000000010000000000000003", - "P00 INFO: new backup label = 20211029-120551F_20211029-120555D", - "P00 INFO: diff backup size = 8KB, file total = 976", + " [filtered 7 lines of output]", + "P00 INFO: check archive for segment(s) 000000010000000000000004:000000010000000000000005", + "P00 INFO: new backup label = 20211231-195532F_20211231-195538D", + "P00 INFO: diff backup size = 8.3KB, file total = 976", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1437-325c4353 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=1357-bbff90c0 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" ] } }, @@ -1144,20 +1146,20 @@ " cipher: aes-256-cbc", "", " db (current)", - " wal archive min/max (12): 000000010000000000000001/000000010000000000000003", + " wal archive min/max (12): 000000010000000000000001/000000010000000000000005", "", - " full backup: 20211029-120551F", - " timestamp start/stop: 2021-10-29 12:05:51 / 2021-10-29 12:05:54", - " wal start/stop: 000000010000000000000002 / 000000010000000000000002", + " full backup: 20211231-195532F", + " timestamp start/stop: 2021-12-31 19:55:32 / 2021-12-31 19:55:36", + " wal start/stop: 000000010000000000000002 / 000000010000000000000003", " database size: 23.4MB, database backup size: 23.4MB", " repo1: backup set size: 2.8MB, backup size: 2.8MB", "", - " diff backup: 20211029-120551F_20211029-120555D", - " timestamp start/stop: 2021-10-29 12:05:55 / 2021-10-29 12:05:56", - " wal start/stop: 000000010000000000000003 / 000000010000000000000003", + " diff backup: 20211231-195532F_20211231-195538D", + " timestamp start/stop: 2021-12-31 19:55:38 / 2021-12-31 19:55:40", + " wal start/stop: 000000010000000000000004 / 000000010000000000000005", " database size: 23.4MB, database backup size: 8.3KB", - " repo1: backup set size: 2.8MB, backup size: 480B", - " backup reference list: 20211029-120551F" + " repo1: backup set size: 2.8MB, backup size: 496B", + " backup reference list: 20211231-195532F" ] } }, @@ -1410,7 +1412,7 @@ "output" : [ " name | last_successful_backup | last_archived_wal ", "--------+------------------------+--------------------------", - " \"demo\" | 2021-10-29 12:05:56+00 | 000000010000000000000003", + " \"demo\" | 2021-12-31 19:55:40+00 | 000000010000000000000005", "(1 row)" ] } @@ -1444,7 +1446,7 @@ "type" : "exe", "value" : { "output" : [ - "1635509156" + "1640980540" ] } }, @@ -1463,7 +1465,7 @@ "type" : "exe", "value" : { "output" : [ - "\"000000010000000000000003\"" + "\"000000010000000000000005\"" ] } }, @@ -1508,7 +1510,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "archive retention on backup 20211029-120551F|remove archive" + "archive retention on backup 20211231-195532F|remove archive" ] }, "host" : "pg-primary", @@ -1519,10 +1521,10 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 984 lines of output]", + " [filtered 985 lines of output]", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1736-ff9ce9ad --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", - "P00 DETAIL: repo1: 12-1 archive retention on backup 20211029-120551F, start = 000000010000000000000002", + "P00 INFO: expire command begin 2.37: --exec-id=1625-0be930da --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", + "P00 DETAIL: repo1: 12-1 archive retention on backup 20211231-195532F, start = 000000010000000000000002", "P00 INFO: repo1: 12-1 remove archive, start = 000000010000000000000001, stop = 000000010000000000000001", "P00 INFO: expire command end: completed successfully" ] @@ -1542,7 +1544,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120606F" + "20211231-195604F" ] } }, @@ -1557,7 +1559,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "expire full backup set 20211029-120551F|archive retention on backup 20211029-120606F|remove archive" + "expire full backup set 20211231-195532F|archive retention on backup 20211231-195604F|remove archive" ] }, "host" : "pg-primary", @@ -1568,13 +1570,13 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 8 lines of output]", + " [filtered 9 lines of output]", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1766-5c7c95b7 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", - "P00 INFO: repo1: expire full backup set 20211029-120551F, 20211029-120551F_20211029-120555D", - "P00 INFO: repo1: remove expired backup 20211029-120551F_20211029-120555D", - "P00 INFO: repo1: remove expired backup 20211029-120551F", - "P00 INFO: repo1: 12-1 remove archive, start = 0000000100000000, stop = 000000020000000000000004", + "P00 INFO: expire command begin 2.37: --exec-id=1652-b7ac48aa --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", + "P00 INFO: repo1: expire full backup set 20211231-195532F, 20211231-195532F_20211231-195538D", + "P00 INFO: repo1: remove expired backup 20211231-195532F_20211231-195538D", + "P00 INFO: repo1: remove expired backup 20211231-195532F", + "P00 INFO: repo1: 12-1 remove archive, start = 0000000100000000, stop = 000000020000000000000006", "P00 INFO: expire command end: completed successfully" ] } @@ -1637,7 +1639,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120610F_20211029-120613D" + "20211231-195609F_20211231-195615D" ] } }, @@ -1665,7 +1667,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "expire diff backup set 20211029-120610F_20211029-120613D" + "expire diff backup set 20211231-195609F_20211231-195615D" ] }, "host" : "pg-primary", @@ -1676,12 +1678,12 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 9 lines of output]", + " [filtered 10 lines of output]", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1848-52792cb8 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-diff=1 --repo1-retention-full=2 --stanza=demo", - "P00 INFO: repo1: expire diff backup set 20211029-120610F_20211029-120613D, 20211029-120610F_20211029-120615I", - "P00 INFO: repo1: remove expired backup 20211029-120610F_20211029-120615I", - "P00 INFO: repo1: remove expired backup 20211029-120610F_20211029-120613D", + "P00 INFO: expire command begin 2.37: --exec-id=1729-c585e461 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-diff=1 --repo1-retention-full=2 --stanza=demo", + "P00 INFO: repo1: expire diff backup set 20211231-195609F_20211231-195615D, 20211231-195609F_20211231-195619I", + "P00 INFO: repo1: remove expired backup 20211231-195609F_20211231-195619I", + "P00 INFO: repo1: remove expired backup 20211231-195609F_20211231-195615D", "P00 INFO: expire command end: completed successfully" ] } @@ -1731,7 +1733,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120610F_20211029-120618D" + "20211231-195609F_20211231-195621D" ] } }, @@ -1772,11 +1774,11 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 5 lines of output]", - "P00 INFO: backup stop archive = 00000002000000000000000D, lsn = 0/D000100", - "P00 INFO: check archive for segment(s) 00000002000000000000000D:00000002000000000000000D", - "P00 INFO: new backup label = 20211029-120610F_20211029-120620D", - "P00 INFO: diff backup size = 8KB, file total = 976", + " [filtered 6 lines of output]", + "P00 INFO: backup stop archive = 000000020000000000000013, lsn = 0/13000050", + "P00 INFO: check archive for segment(s) 000000020000000000000012:000000020000000000000013", + "P00 INFO: new backup label = 20211231-195609F_20211231-195627D", + "P00 INFO: diff backup size = 8.3KB, file total = 976", "P00 INFO: backup command end: completed successfully", " [filtered 2 lines of output]" ] @@ -1796,7 +1798,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120610F_20211029-120620D" + "20211231-195609F_20211231-195627D" ] } }, @@ -1811,7 +1813,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "archive retention on backup 20211029-120610F_20211029-120618D|remove archive" + "archive retention on backup 20211231-195609F_20211231-195621D|remove archive" ] }, "host" : "pg-primary", @@ -1822,13 +1824,13 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: expire command begin 2.36: --exec-id=1939-1405cf84 --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-archive=1 --repo1-retention-archive-type=diff --repo1-retention-diff=2 --repo1-retention-full=2 --stanza=demo", - "P00 DETAIL: repo1: 12-1 archive retention on backup 20211029-120606F, start = 000000020000000000000005, stop = 000000020000000000000005", - "P00 DETAIL: repo1: 12-1 archive retention on backup 20211029-120610F, start = 000000020000000000000006, stop = 000000020000000000000006", - "P00 DETAIL: repo1: 12-1 archive retention on backup 20211029-120610F_20211029-120618D, start = 00000002000000000000000A, stop = 00000002000000000000000A", - "P00 DETAIL: repo1: 12-1 archive retention on backup 20211029-120610F_20211029-120620D, start = 00000002000000000000000D", - "P00 INFO: repo1: 12-1 remove archive, start = 000000020000000000000007, stop = 000000020000000000000009", - "P00 INFO: repo1: 12-1 remove archive, start = 00000002000000000000000B, stop = 00000002000000000000000C", + "P00 INFO: expire command begin 2.37: --exec-id=1812-3bf5f31c --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-archive=1 --repo1-retention-archive-type=diff --repo1-retention-diff=2 --repo1-retention-full=2 --stanza=demo", + "P00 DETAIL: repo1: 12-1 archive retention on backup 20211231-195604F, start = 000000020000000000000007, stop = 000000020000000000000007", + "P00 DETAIL: repo1: 12-1 archive retention on backup 20211231-195609F, start = 000000020000000000000008, stop = 000000020000000000000009", + "P00 DETAIL: repo1: 12-1 archive retention on backup 20211231-195609F_20211231-195621D, start = 00000002000000000000000E, stop = 00000002000000000000000F", + "P00 DETAIL: repo1: 12-1 archive retention on backup 20211231-195609F_20211231-195627D, start = 000000020000000000000012", + "P00 INFO: repo1: 12-1 remove archive, start = 00000002000000000000000A, stop = 00000002000000000000000D", + "P00 INFO: repo1: 12-1 remove archive, start = 000000020000000000000010, stop = 000000020000000000000011", "P00 INFO: expire command end: completed successfully" ] } @@ -1874,7 +1876,7 @@ "P00 INFO: remove invalid files/links/paths from '/var/lib/postgresql/12/demo'", "P00 DETAIL: remove invalid file '/var/lib/postgresql/12/demo/backup_label.old'", "P00 DETAIL: remove invalid file '/var/lib/postgresql/12/demo/base/1/pg_internal.init'", - " [filtered 815 lines of output]", + " [filtered 816 lines of output]", "P01 DETAIL: restore file /var/lib/postgresql/12/demo/base/13397/PG_VERSION - exists and matches backup (3B, 99%) checksum ad552e6dc057d1d825bf49df79d6b98eba846ebe", "P01 DETAIL: restore file /var/lib/postgresql/12/demo/base/1/PG_VERSION - exists and matches backup (3B, 99%) checksum ad552e6dc057d1d825bf49df79d6b98eba846ebe", "P01 DETAIL: restore file /var/lib/postgresql/12/demo/PG_VERSION - exists and matches backup (3B, 100%) checksum ad552e6dc057d1d825bf49df79d6b98eba846ebe", @@ -2047,7 +2049,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120610F_20211029-120629I" + "20211231-195609F_20211231-195642I" ] } }, @@ -2056,7 +2058,7 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo \\", - " --set=20211029-120610F_20211029-120629I info" + " --set=20211231-195609F_20211231-195642I info" ], "highlight" : { "filter" : true, @@ -2075,7 +2077,7 @@ "output" : [ " [filtered 11 lines of output]", " repo1: backup set size: 4.7MB, backup size: 1.9MB", - " backup reference list: 20211029-120610F, 20211029-120610F_20211029-120620D", + " backup reference list: 20211231-195609F, 20211231-195609F_20211231-195627D", " database list: postgres (13398), test1 (24576), test2 (24577)" ] } @@ -2319,7 +2321,7 @@ "type" : "exe", "value" : { "output" : [ - "2021-10-29 12:06:41.23673+00" + "2021-12-31 19:57:03.172377+00" ] } }, @@ -2385,7 +2387,7 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo --delta \\", - " --type=time \"--target=2021-10-29 12:06:41.23673+00\" \\", + " --type=time \"--target=2021-12-31 19:57:03.172377+00\" \\", " --target-action=promote restore" ], "host" : "pg-primary", @@ -2430,9 +2432,9 @@ "value" : { "output" : [ " [filtered 14 lines of output]", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:43", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:57:07", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", - "recovery_target_time = '2021-10-29 12:06:41.23673+00'", + "recovery_target_time = '2021-12-31 19:57:03.172377+00'", "recovery_target_action = 'promote'" ] } @@ -2514,16 +2516,16 @@ "output" : [ " [filtered 3 lines of output]", "LOG: listening on Unix socket \"/var/run/postgresql/.s.PGSQL.5432\"", - "LOG: database system was interrupted; last known up at 2021-10-29 12:06:38 UTC", - "LOG: starting point-in-time recovery to 2021-10-29 12:06:41.23673+00", + "LOG: database system was interrupted; last known up at 2021-12-31 19:56:58 UTC", + "LOG: starting point-in-time recovery to 2021-12-31 19:57:03.172377+00", "LOG: restored log file \"00000004.history\" from archive", - "LOG: restored log file \"000000040000000000000010\" from archive", + "LOG: restored log file \"000000040000000000000016\" from archive", " [filtered 2 lines of output]", "LOG: database system is ready to accept read only connections", - "LOG: restored log file \"000000040000000000000011\" from archive", - "LOG: recovery stopping before commit of transaction 495, time 2021-10-29 12:06:42.508122+00", - "LOG: redo done at 0/11019DD8", - "LOG: last completed transaction was at log time 2021-10-29 12:06:39.99863+00", + "LOG: restored log file \"000000040000000000000017\" from archive", + "LOG: recovery stopping before commit of transaction 495, time 2021-12-31 19:57:05.396152+00", + "LOG: redo done at 0/17019E10", + "LOG: last completed transaction was at log time 2021-12-31 19:57:00.903751+00", "LOG: selected new timeline ID: 5", "LOG: archive recovery complete", " [filtered 2 lines of output]" @@ -2588,7 +2590,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120610F_20211029-120649I" + "20211231-195609F_20211231-195716I" ] } }, @@ -2602,7 +2604,7 @@ "filter" : false, "filter-context" : 2, "list" : [ - "20211029-120610F_20211029-120649I" + "20211231-195609F_20211231-195716I" ] }, "host" : "pg-primary", @@ -2618,47 +2620,47 @@ " cipher: aes-256-cbc", "", " db (current)", - " wal archive min/max (12): 000000020000000000000005/000000050000000000000012", + " wal archive min/max (12): 000000020000000000000007/000000050000000000000018", "", - " full backup: 20211029-120606F", - " timestamp start/stop: 2021-10-29 12:06:06 / 2021-10-29 12:06:09", - " wal start/stop: 000000020000000000000005 / 000000020000000000000005", + " full backup: 20211231-195604F", + " timestamp start/stop: 2021-12-31 19:56:04 / 2021-12-31 19:56:07", + " wal start/stop: 000000020000000000000007 / 000000020000000000000007", " database size: 23.4MB, database backup size: 23.4MB", " repo1: backup set size: 2.8MB, backup size: 2.8MB", "", - " full backup: 20211029-120610F", - " timestamp start/stop: 2021-10-29 12:06:10 / 2021-10-29 12:06:12", - " wal start/stop: 000000020000000000000006 / 000000020000000000000006", + " full backup: 20211231-195609F", + " timestamp start/stop: 2021-12-31 19:56:09 / 2021-12-31 19:56:13", + " wal start/stop: 000000020000000000000008 / 000000020000000000000009", " database size: 23.4MB, database backup size: 23.4MB", " repo1: backup set size: 2.8MB, backup size: 2.8MB", "", - " diff backup: 20211029-120610F_20211029-120620D", - " timestamp start/stop: 2021-10-29 12:06:20 / 2021-10-29 12:06:22", - " wal start/stop: 00000002000000000000000D / 00000002000000000000000D", + " diff backup: 20211231-195609F_20211231-195627D", + " timestamp start/stop: 2021-12-31 19:56:27 / 2021-12-31 19:56:29", + " wal start/stop: 000000020000000000000012 / 000000020000000000000013", " database size: 23.4MB, database backup size: 8.3KB", - " repo1: backup set size: 2.8MB, backup size: 496B", - " backup reference list: 20211029-120610F", + " repo1: backup set size: 2.8MB, backup size: 512B", + " backup reference list: 20211231-195609F", "", - " incr backup: 20211029-120610F_20211029-120629I", - " timestamp start/stop: 2021-10-29 12:06:29 / 2021-10-29 12:06:30", - " wal start/stop: 00000003000000000000000F / 00000003000000000000000F", + " incr backup: 20211231-195609F_20211231-195642I", + " timestamp start/stop: 2021-12-31 19:56:42 / 2021-12-31 19:56:44", + " wal start/stop: 000000030000000000000015 / 000000030000000000000015", " database size: 38.7MB, database backup size: 15.8MB", " repo1: backup set size: 4.7MB, backup size: 1.9MB", - " backup reference list: 20211029-120610F, 20211029-120610F_20211029-120620D", + " backup reference list: 20211231-195609F, 20211231-195609F_20211231-195627D", "", - " diff backup: 20211029-120610F_20211029-120638D", - " timestamp start/stop: 2021-10-29 12:06:38 / 2021-10-29 12:06:39", - " wal start/stop: 000000040000000000000010 / 000000040000000000000010", + " diff backup: 20211231-195609F_20211231-195658D", + " timestamp start/stop: 2021-12-31 19:56:58 / 2021-12-31 19:57:00", + " wal start/stop: 000000040000000000000016 / 000000040000000000000016", " database size: 31MB, database backup size: 8.2MB", " repo1: backup set size: 3.8MB, backup size: 1011.1KB", - " backup reference list: 20211029-120610F", + " backup reference list: 20211231-195609F", "", - " incr backup: 20211029-120610F_20211029-120649I", - " timestamp start/stop: 2021-10-29 12:06:49 / 2021-10-29 12:06:50", - " wal start/stop: 000000050000000000000012 / 000000050000000000000012", + " incr backup: 20211231-195609F_20211231-195716I", + " timestamp start/stop: 2021-12-31 19:57:16 / 2021-12-31 19:57:18", + " wal start/stop: 000000050000000000000018 / 000000050000000000000018", " database size: 31MB, database backup size: 2.2MB", " repo1: backup set size: 3.8MB, backup size: 234KB", - " backup reference list: 20211029-120610F, 20211029-120610F_20211029-120638D" + " backup reference list: 20211231-195609F, 20211231-195609F_20211231-195658D" ] } }, @@ -2680,8 +2682,8 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo --delta \\", - " --set=20211029-120610F_20211029-120649I \\", - " --type=time \"--target=2021-10-29 12:06:41.23673+00\" --target-action=promote restore" + " --set=20211231-195609F_20211231-195716I \\", + " --type=time \"--target=2021-12-31 19:57:03.172377+00\" --target-action=promote restore" ], "host" : "pg-primary", "load-env" : true, @@ -2780,14 +2782,14 @@ "output" : [ " [filtered 3 lines of output]", "LOG: listening on Unix socket \"/var/run/postgresql/.s.PGSQL.5432\"", - "LOG: database system was interrupted; last known up at 2021-10-29 12:06:49 UTC", - "LOG: starting point-in-time recovery to 2021-10-29 12:06:41.23673+00", + "LOG: database system was interrupted; last known up at 2021-12-31 19:57:17 UTC", + "LOG: starting point-in-time recovery to 2021-12-31 19:57:03.172377+00", "LOG: restored log file \"00000005.history\" from archive", - "LOG: restored log file \"000000050000000000000012\" from archive", - "LOG: redo starts at 0/12000028", - "LOG: consistent recovery state reached at 0/12000100", + "LOG: restored log file \"000000050000000000000018\" from archive", + "LOG: redo starts at 0/18000028", + "LOG: consistent recovery state reached at 0/18000100", "LOG: database system is ready to accept read only connections", - "LOG: redo done at 0/12000100", + "LOG: redo done at 0/18000100", " [filtered 7 lines of output]" ] } @@ -2810,7 +2812,7 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo --delta \\", - " --type=time \"--target=2021-10-29 12:06:41.23673+00\" \\", + " --type=time \"--target=2021-12-31 19:57:03.172377+00\" \\", " --target-action=promote restore" ], "host" : "pg-primary", @@ -2911,15 +2913,15 @@ " [filtered 5 lines of output]", "LOG: restored log file \"00000005.history\" from archive", "LOG: restored log file \"00000006.history\" from archive", - "LOG: starting point-in-time recovery to 2021-10-29 12:06:41.23673+00", + "LOG: starting point-in-time recovery to 2021-12-31 19:57:03.172377+00", "LOG: restored log file \"00000006.history\" from archive", - "LOG: restored log file \"000000040000000000000010\" from archive", + "LOG: restored log file \"000000040000000000000016\" from archive", " [filtered 4 lines of output]", "LOG: database system is ready to accept read only connections", - "LOG: restored log file \"000000050000000000000011\" from archive", - "LOG: recovery stopping before commit of transaction 496, time 2021-10-29 12:06:48.410664+00", - "LOG: redo done at 0/11022408", - "LOG: last completed transaction was at log time 2021-10-29 12:06:39.99863+00", + "LOG: restored log file \"000000050000000000000017\" from archive", + "LOG: recovery stopping before commit of transaction 496, time 2021-12-31 19:57:15.721973+00", + "LOG: redo done at 0/17022440", + "LOG: last completed transaction was at log time 2021-12-31 19:57:00.903751+00", "LOG: selected new timeline ID: 7", "LOG: archive recovery complete", " [filtered 2 lines of output]" @@ -3058,16 +3060,16 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: backup command begin 2.36: --exec-id=2798-0931aac5 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=4 --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo --start-fast", + "P00 INFO: backup command begin 2.37: --exec-id=2571-9dfe952d --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=4 --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo --start-fast", "P00 WARN: no prior backup exists, incr backup has been changed to full", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", - "P00 INFO: backup start archive = 000000070000000000000012, lsn = 0/12000028", - " [filtered 2 lines of output]", - "P00 INFO: check archive for segment(s) 000000070000000000000012:000000070000000000000012", - "P00 INFO: new backup label = 20211029-120704F", + "P00 INFO: backup start archive = 000000070000000000000018, lsn = 0/18000028", + " [filtered 3 lines of output]", + "P00 INFO: check archive for segment(s) 000000070000000000000018:000000070000000000000018", + "P00 INFO: new backup label = 20211231-195743F", "P00 INFO: full backup size = 31MB, file total = 1282", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=2798-0931aac5 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=2571-9dfe952d --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo" ] } }, @@ -3217,16 +3219,16 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: backup command begin 2.36: --exec-id=2873-6aad5a60 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=4 --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo --start-fast", + "P00 INFO: backup command begin 2.37: --exec-id=2636-5e500c5a --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=4 --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo --start-fast", "P00 WARN: no prior backup exists, incr backup has been changed to full", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", - "P00 INFO: backup start archive = 000000070000000000000013, lsn = 0/13000028", - " [filtered 2 lines of output]", - "P00 INFO: check archive for segment(s) 000000070000000000000013:000000070000000000000013", - "P00 INFO: new backup label = 20211029-120709F", + "P00 INFO: backup start archive = 000000070000000000000019, lsn = 0/19000028", + " [filtered 3 lines of output]", + "P00 INFO: check archive for segment(s) 000000070000000000000019:00000007000000000000001A", + "P00 INFO: new backup label = 20211231-195800F", "P00 INFO: full backup size = 31MB, file total = 1282", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=2873-6aad5a60 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=2636-5e500c5a --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo" ] } }, @@ -3326,7 +3328,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stop command begin 2.36: --exec-id=2937-e18e99b9 --log-level-console=info --log-level-stderr=off --no-log-timestamp --stanza=demo", + "P00 INFO: stop command begin 2.37: --exec-id=2684-91d01886 --log-level-console=info --log-level-stderr=off --no-log-timestamp --stanza=demo", "P00 INFO: stop command end: completed successfully" ] } @@ -3353,7 +3355,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stanza-delete command begin 2.36: --exec-id=2947-b29fa5d9 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo=1 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo4-gcs-bucket=demo-bucket --repo4-gcs-key= --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo4-path=/demo-repo --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --repo4-type=gcs --stanza=demo", + "P00 INFO: stanza-delete command begin 2.37: --exec-id=2692-aa1d1fa5 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo=1 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo4-gcs-bucket=demo-bucket --repo4-gcs-key= --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo4-path=/demo-repo --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --repo4-type=gcs --stanza=demo", "P00 INFO: stanza-delete command end: completed successfully" ] } @@ -3416,7 +3418,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo scp build:/build/pgbackrest-release-2.36/src/pgbackrest /usr/bin" + "sudo scp build:/build/pgbackrest-release-2.37/src/pgbackrest /usr/bin" ], "cmd-extra" : "2>&1", "host" : "repository", @@ -3973,17 +3975,17 @@ " cipher: none", "", " db (current)", - " wal archive min/max (12): 00000008000000000000001A/00000008000000000000001B", + " wal archive min/max (12): 000000080000000000000020/000000080000000000000022", "", - " full backup: 20211029-120743F", - " timestamp start/stop: 2021-10-29 12:07:43 / 2021-10-29 12:07:46", - " wal start/stop: 00000008000000000000001A / 00000008000000000000001A", + " full backup: 20211231-195910F", + " timestamp start/stop: 2021-12-31 19:59:10 / 2021-12-31 19:59:15", + " wal start/stop: 000000080000000000000020 / 000000080000000000000020", " database size: 31MB, database backup size: 31MB", " repo1: backup set size: 3.7MB, backup size: 3.7MB", "", - " full backup: 20211029-120748F", - " timestamp start/stop: 2021-10-29 12:07:48 / 2021-10-29 12:07:50", - " wal start/stop: 00000008000000000000001B / 00000008000000000000001B", + " full backup: 20211231-195919F", + " timestamp start/stop: 2021-12-31 19:59:19 / 2021-12-31 19:59:25", + " wal start/stop: 000000080000000000000022 / 000000080000000000000022", " database size: 31MB, database backup size: 31MB", " repo1: backup set size: 3.7MB, backup size: 3.7MB" ] @@ -4145,7 +4147,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo scp build:/build/pgbackrest-release-2.36/src/pgbackrest /usr/bin" + "sudo scp build:/build/pgbackrest-release-2.37/src/pgbackrest /usr/bin" ], "cmd-extra" : "2>&1", "host" : "pg-standby", @@ -4427,26 +4429,26 @@ "# Do not edit this file manually!", "# It will be overwritten by the ALTER SYSTEM command.", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:05:58", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:55:45", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:23", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:56:32", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:32", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:56:49", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", - "# Removed by pgBackRest restore on 2021-10-29 12:06:56 # recovery_target = 'immediate'", - "# Removed by pgBackRest restore on 2021-10-29 12:06:56 # recovery_target_action = 'promote'", + "# Removed by pgBackRest restore on 2021-12-31 19:57:31 # recovery_target = 'immediate'", + "# Removed by pgBackRest restore on 2021-12-31 19:57:31 # recovery_target_action = 'promote'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:56", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:57:31", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", - "# Removed by pgBackRest restore on 2021-10-29 12:07:38 # recovery_target_time = '2021-10-29 12:06:41.23673+00'", - "# Removed by pgBackRest restore on 2021-10-29 12:07:38 # recovery_target_action = 'promote'", + "# Removed by pgBackRest restore on 2021-12-31 19:59:02 # recovery_target_time = '2021-12-31 19:57:03.172377+00'", + "# Removed by pgBackRest restore on 2021-12-31 19:59:02 # recovery_target_action = 'promote'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:07:38", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:59:02", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:08:01", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 20:00:00", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'" ] } @@ -4567,12 +4569,12 @@ "output" : [ " [filtered 4 lines of output]", "LOG: listening on Unix socket \"/var/run/postgresql/.s.PGSQL.5432\"", - "LOG: database system was interrupted; last known up at 2021-10-29 12:07:48 UTC", + "LOG: database system was interrupted; last known up at 2021-12-31 19:59:19 UTC", "LOG: entering standby mode", "LOG: restored log file \"00000008.history\" from archive", - "LOG: restored log file \"00000008000000000000001B\" from archive", - "LOG: redo starts at 0/1B000028", - "LOG: consistent recovery state reached at 0/1B000100", + "LOG: restored log file \"000000080000000000000022\" from archive", + "LOG: redo starts at 0/22000028", + "LOG: consistent recovery state reached at 0/22000100", "LOG: database system is ready to accept read only connections" ] } @@ -4654,7 +4656,7 @@ "output" : [ " pg_switch_wal | current_timestamp ", "---------------+-------------------------------", - " 0/1C021718 | 2021-10-29 12:08:07.549186+00", + " 0/23021750 | 2021-12-31 20:00:14.253018+00", "(1 row)" ] } @@ -4683,7 +4685,7 @@ "output" : [ " message | current_timestamp ", "----------------+-------------------------------", - " Important Data | 2021-10-29 12:08:08.414679+00", + " Important Data | 2021-12-31 20:00:17.565198+00", "(1 row)" ] } @@ -4709,7 +4711,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: check command begin 2.36: --exec-id=1417-21ffb037 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-host=repository --stanza=demo", + "P00 INFO: check command begin 2.37: --exec-id=1338-239cb6ec --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-host=repository --stanza=demo", "P00 INFO: check repo1 (standby)", "P00 INFO: switch wal not performed because this is a standby", "P00 INFO: check command end: completed successfully" @@ -4859,26 +4861,26 @@ "# Do not edit this file manually!", "# It will be overwritten by the ALTER SYSTEM command.", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:05:58", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:55:45", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:23", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:56:32", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:32", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:56:49", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", - "# Removed by pgBackRest restore on 2021-10-29 12:06:56 # recovery_target = 'immediate'", - "# Removed by pgBackRest restore on 2021-10-29 12:06:56 # recovery_target_action = 'promote'", + "# Removed by pgBackRest restore on 2021-12-31 19:57:31 # recovery_target = 'immediate'", + "# Removed by pgBackRest restore on 2021-12-31 19:57:31 # recovery_target_action = 'promote'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:06:56", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:57:31", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", - "# Removed by pgBackRest restore on 2021-10-29 12:07:38 # recovery_target_time = '2021-10-29 12:06:41.23673+00'", - "# Removed by pgBackRest restore on 2021-10-29 12:07:38 # recovery_target_action = 'promote'", + "# Removed by pgBackRest restore on 2021-12-31 19:59:02 # recovery_target_time = '2021-12-31 19:57:03.172377+00'", + "# Removed by pgBackRest restore on 2021-12-31 19:59:02 # recovery_target_action = 'promote'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:07:38", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:59:02", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "", - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:08:11", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 20:00:26", "primary_conninfo = 'host=172.17.0.5 port=5432 user=replicator'", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'" ] @@ -4946,8 +4948,8 @@ "output" : [ " [filtered 11 lines of output]", "LOG: database system is ready to accept read only connections", - "LOG: restored log file \"00000008000000000000001C\" from archive", - "LOG: started streaming WAL from primary at 0/1D000000 on timeline 8" + "LOG: restored log file \"000000080000000000000023\" from archive", + "LOG: started streaming WAL from primary at 0/24000000 on timeline 8" ] } }, @@ -4979,7 +4981,7 @@ "output" : [ " message | current_timestamp ", "----------------+-------------------------------", - " Important Data | 2021-10-29 12:08:16.746025+00", + " Important Data | 2021-12-31 20:00:34.575889+00", "(1 row)" ] } @@ -5006,9 +5008,9 @@ "type" : "exe", "value" : { "output" : [ - " message | current_timestamp ", - "----------------+------------------------------", - " Important Data | 2021-10-29 12:08:16.93526+00", + " message | current_timestamp ", + "----------------+-------------------------------", + " Important Data | 2021-12-31 20:00:35.482415+00", "(1 row)" ] } @@ -5239,10 +5241,10 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: check command begin 2.36: --exec-id=3602-14941921 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-host=repository --stanza=demo", + "P00 INFO: check command begin 2.37: --exec-id=3279-1d1209e9 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --repo1-host=repository --stanza=demo", "P00 INFO: check repo1 configuration (primary)", "P00 INFO: check repo1 archive for WAL (primary)", - "P00 INFO: WAL segment 000000080000000000000022 successfully archived to '/var/lib/pgbackrest/archive/demo/12-1/0000000800000000/000000080000000000000022-256032fd13a8e92807d2fbf916d91843a34c484f.gz' on repo1", + "P00 INFO: WAL segment 000000080000000000000029 successfully archived to '/var/lib/pgbackrest/archive/demo/12-1/0000000800000000/000000080000000000000029-45c70258c3af934d29956e009b6f211e949dcb1a.gz' on repo1", "P00 INFO: check command end: completed successfully" ] } @@ -5269,24 +5271,24 @@ "value" : { "output" : [ "-------------------PROCESS START-------------------", - "P00 INFO: archive-push:async command begin 2.36: [/var/lib/postgresql/12/demo/pg_wal] --archive-async --exec-id=3587-0eaf57f4 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: push 1 WAL file(s) to archive: 00000008000000000000001D", - "P01 DETAIL: pushed WAL file '00000008000000000000001D' to the archive", + "P00 INFO: archive-push:async command begin 2.37: [/var/lib/postgresql/12/demo/pg_wal] --archive-async --exec-id=3265-085154e0 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: push 1 WAL file(s) to archive: 000000080000000000000024", + "P01 DETAIL: pushed WAL file '000000080000000000000024' to the archive", "P00 INFO: archive-push:async command end: completed successfully", "", "-------------------PROCESS START-------------------", - "P00 INFO: archive-push:async command begin 2.36: [/var/lib/postgresql/12/demo/pg_wal] --archive-async --exec-id=3605-804d267f --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: push 4 WAL file(s) to archive: 00000008000000000000001E...000000080000000000000021", - "P01 DETAIL: pushed WAL file '00000008000000000000001E' to the archive", - "P02 DETAIL: pushed WAL file '00000008000000000000001F' to the archive", - "P02 DETAIL: pushed WAL file '000000080000000000000021' to the archive", - "P01 DETAIL: pushed WAL file '000000080000000000000020' to the archive", + "P00 INFO: archive-push:async command begin 2.37: [/var/lib/postgresql/12/demo/pg_wal] --archive-async --exec-id=3283-c2b7f5b4 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: push 4 WAL file(s) to archive: 000000080000000000000025...000000080000000000000028", + "P02 DETAIL: pushed WAL file '000000080000000000000026' to the archive", + "P01 DETAIL: pushed WAL file '000000080000000000000025' to the archive", + "P02 DETAIL: pushed WAL file '000000080000000000000027' to the archive", + "P01 DETAIL: pushed WAL file '000000080000000000000028' to the archive", "P00 INFO: archive-push:async command end: completed successfully", "", "-------------------PROCESS START-------------------", - "P00 INFO: archive-push:async command begin 2.36: [/var/lib/postgresql/12/demo/pg_wal] --archive-async --exec-id=3621-7c1ac38a --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: push 1 WAL file(s) to archive: 000000080000000000000022", - "P01 DETAIL: pushed WAL file '000000080000000000000022' to the archive", + "P00 INFO: archive-push:async command begin 2.37: [/var/lib/postgresql/12/demo/pg_wal] --archive-async --exec-id=3299-b52ab60e --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: push 1 WAL file(s) to archive: 000000080000000000000029", + "P01 DETAIL: pushed WAL file '000000080000000000000029' to the archive", "P00 INFO: archive-push:async command end: completed successfully" ] } @@ -5326,24 +5328,34 @@ "value" : { "output" : [ "-------------------PROCESS START-------------------", - "P00 INFO: archive-get:async command begin 2.36: [00000008000000000000001B, 00000008000000000000001C, 00000008000000000000001D, 00000008000000000000001E, 00000008000000000000001F, 000000080000000000000020, 000000080000000000000021, 000000080000000000000022] --archive-async --exec-id=1652-36029fb1 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: get 8 WAL file(s) from archive: 00000008000000000000001B...000000080000000000000022", - "P02 DETAIL: found 00000008000000000000001C in the repo1: 12-1 archive", - "P01 DETAIL: found 00000008000000000000001B in the repo1: 12-1 archive", - "P00 DETAIL: unable to find 00000008000000000000001D in the archive", + "P00 INFO: archive-get:async command begin 2.37: [000000080000000000000022, 000000080000000000000023, 000000080000000000000024, 000000080000000000000025, 000000080000000000000026, 000000080000000000000027, 000000080000000000000028, 000000080000000000000029] --archive-async --exec-id=1548-7fe4e456 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000022...000000080000000000000029", + "P01 DETAIL: found 000000080000000000000022 in the repo1: 12-1 archive", + "P02 DETAIL: found 000000080000000000000023 in the repo1: 12-1 archive", + "P00 DETAIL: unable to find 000000080000000000000024 in the archive", "P00 INFO: archive-get:async command end: completed successfully", " [filtered 14 lines of output]", - "P00 INFO: archive-get:async command begin 2.36: [00000008000000000000001D, 00000008000000000000001E, 00000008000000000000001F, 000000080000000000000020, 000000080000000000000021, 000000080000000000000022, 000000080000000000000023, 000000080000000000000024] --archive-async --exec-id=1694-87354168 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: get 8 WAL file(s) from archive: 00000008000000000000001D...000000080000000000000024", - "P02 DETAIL: found 00000008000000000000001E in the repo1: 12-1 archive", - "P02 DETAIL: found 00000008000000000000001F in the repo1: 12-1 archive", - "P01 DETAIL: found 00000008000000000000001D in the repo1: 12-1 archive", - "P02 DETAIL: found 000000080000000000000020 in the repo1: 12-1 archive", - "P01 DETAIL: found 000000080000000000000021 in the repo1: 12-1 archive", - "P02 DETAIL: found 000000080000000000000022 in the repo1: 12-1 archive", - "P00 DETAIL: unable to find 000000080000000000000023 in the archive", + "P00 INFO: archive-get:async command begin 2.37: [000000080000000000000024, 000000080000000000000025, 000000080000000000000026, 000000080000000000000027, 000000080000000000000028, 000000080000000000000029, 00000008000000000000002A, 00000008000000000000002B] --archive-async --exec-id=1590-245d7f0f --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000024...00000008000000000000002B", + "P01 DETAIL: found 000000080000000000000024 in the repo1: 12-1 archive", + "P00 DETAIL: unable to find 000000080000000000000025 in the archive", + "P00 INFO: archive-get:async command end: completed successfully", + " [filtered 2 lines of output]", + "P00 INFO: archive-get:async command begin 2.37: [000000080000000000000025, 000000080000000000000026, 000000080000000000000027, 000000080000000000000028, 000000080000000000000029, 00000008000000000000002A, 00000008000000000000002B, 00000008000000000000002C] --archive-async --exec-id=1598-571f7ef0 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000025...00000008000000000000002C", + "P02 DETAIL: found 000000080000000000000026 in the repo1: 12-1 archive", + "P01 DETAIL: found 000000080000000000000025 in the repo1: 12-1 archive", + "P00 DETAIL: unable to find 000000080000000000000027 in the archive", "P00 INFO: archive-get:async command end: completed successfully", - " [filtered 11 lines of output]" + " [filtered 2 lines of output]", + "P00 INFO: archive-get:async command begin 2.37: [000000080000000000000027, 000000080000000000000028, 000000080000000000000029, 00000008000000000000002A, 00000008000000000000002B, 00000008000000000000002C, 00000008000000000000002D, 00000008000000000000002E] --archive-async --exec-id=1607-3bece536 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/postgresql/12/demo --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000027...00000008000000000000002E", + "P02 DETAIL: found 000000080000000000000028 in the repo1: 12-1 archive", + "P01 DETAIL: found 000000080000000000000027 in the repo1: 12-1 archive", + "P02 DETAIL: found 000000080000000000000029 in the repo1: 12-1 archive", + "P00 DETAIL: unable to find 00000008000000000000002A in the archive", + "P00 INFO: archive-get:async command end: completed successfully", + " [filtered 17 lines of output]" ] } }, @@ -5426,20 +5438,15 @@ "output" : [ " [filtered 2 lines of output]", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", - "P00 INFO: backup start archive = 000000080000000000000024, lsn = 0/24000028", - "P00 INFO: wait for replay on the standby to reach 0/24000028", - "P00 INFO: replay on the standby reached 0/24000028", - "P02 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/2608 (456KB, 19%) checksum 8a1b4147f0cd9e6232ebcf971f34a1b726af93c5", - "P02 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/2673 (280KB, 31%) checksum 13cf784264de5d300e755082fe0d91be52847cb5", - " [filtered 14 lines of output]", - "P02 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/2679 (16KB, 62%) checksum 6405582b3965a1cc4cefa4d89a075bf392bfee62", - "P02 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/2678 (16KB, 63%) checksum f7c84518351ccdb7384c91142454a10665eb4985", - "P01 DETAIL: backup file pg-primary:/var/lib/postgresql/12/demo/global/pg_control (8KB, 63%) checksum 221cf7dcd1912b900bc386e092297f2b3e4b1068", - "P02 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/pg_xact/0000 (8KB, 63%) checksum 32d68ad3e1925c9bfa1fff50781196c74861f1b3", - "P01 DETAIL: backup file pg-primary:/var/lib/postgresql/12/demo/pg_logical/replorigin_checkpoint (8B, 63%) checksum 347fc8f2df71bd4436e38bd1516ccd7ea0d46532", - "P03 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/1249 (440KB, 82%) checksum b7e359434670a0611273a3626a1feb0250243414", - "P04 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/2674 (344KB, 97%) checksum 28827539e9e6b01b680e50390c21331a0a5d0639", - " [filtered 1274 lines of output]" + "P00 INFO: backup start archive = 00000008000000000000002B, lsn = 0/2B000028", + "P00 INFO: wait for replay on the standby to reach 0/2B000028", + "P00 INFO: replay on the standby reached 0/2B000028", + "P00 INFO: check archive for prior segment 00000008000000000000002A", + "P01 DETAIL: backup file pg-primary:/var/lib/postgresql/12/demo/global/pg_control (8KB, 0%) checksum 35e8a0c90ca17e4ad572c965bc503b3559890160", + "P01 DETAIL: backup file pg-primary:/var/lib/postgresql/12/demo/pg_logical/replorigin_checkpoint (8B, 0%) checksum 347fc8f2df71bd4436e38bd1516ccd7ea0d46532", + "P02 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/2608 (456KB, 19%) checksum c5379fa0a0da3e312f114d08ef381e27802113f2", + "P03 DETAIL: backup file pg-standby:/var/lib/postgresql/12/demo/base/13398/1249 (440KB, 38%) checksum 276c6ea0633afcf173bcb10ac3d0f6a1936b9502", + " [filtered 1293 lines of output]" ] } }, @@ -5714,7 +5721,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stanza-upgrade command begin 2.36: --exec-id=4022-d6ee7d0e --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --no-online --pg1-path=/var/lib/postgresql/13/demo --repo1-host=repository --stanza=demo", + "P00 INFO: stanza-upgrade command begin 2.37: --exec-id=3670-e88b2f92 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --no-online --pg1-path=/var/lib/postgresql/13/demo --repo1-host=repository --stanza=demo", "P00 INFO: stanza-upgrade for stanza 'demo' on repo1", "P00 INFO: stanza-upgrade command end: completed successfully" ] @@ -5983,7 +5990,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo mkdir -p /build/pgbackrest-release-2.36" + "sudo mkdir -p /build/pgbackrest-release-2.37" ], "host" : "build", "load-env" : true, @@ -5996,7 +6003,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo cp -r /pgbackrest/src /build/pgbackrest-release-2.36" + "sudo cp -r /pgbackrest/src /build/pgbackrest-release-2.37" ], "host" : "build", "load-env" : true, @@ -6009,7 +6016,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo chown -R vagrant /build/pgbackrest-release-2.36" + "sudo chown -R vagrant /build/pgbackrest-release-2.37" ], "host" : "build", "load-env" : true, @@ -6037,7 +6044,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "cd /build/pgbackrest-release-2.36/src && ./configure && make" + "cd /build/pgbackrest-release-2.37/src && ./configure && make" ], "cmd-extra" : "-j 4", "host" : "build", @@ -6079,7 +6086,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo scp build:/build/pgbackrest-release-2.36/src/pgbackrest /usr/bin" + "sudo scp build:/build/pgbackrest-release-2.37/src/pgbackrest /usr/bin" ], "cmd-extra" : "2>&1", "host" : "pg-primary", @@ -6207,7 +6214,7 @@ "type" : "exe", "value" : { "output" : [ - "pgBackRest 2.36 - General help", + "pgBackRest 2.37 - General help", "", "Usage:", " pgbackrest [options] [command]", @@ -6223,6 +6230,8 @@ " repo-get Get a file from a repository.", " repo-ls List files in a repository.", " restore Restore a database cluster.", + " server pgBackRest server.", + " server-ping Ping pgBackRest server.", " stanza-create Create the required stanza data.", " stanza-delete Delete a stanza.", " stanza-upgrade Upgrade a stanza.", @@ -6366,7 +6375,7 @@ "type" : "exe", "value" : { "output" : [ - "pgBackRest 2.36 - 'backup' command - 'log-path' option help", + "pgBackRest 2.37 - 'backup' command - 'log-path' option help", "", "Path where log files are stored.", "", @@ -6604,7 +6613,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stanza-create command begin 2.36: --exec-id=1109-84785859 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", + "P00 INFO: stanza-create command begin 2.37: --exec-id=952-6f56cb91 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", "P00 INFO: stanza-create for stanza 'demo' on repo1", "P00 INFO: stanza-create command end: completed successfully" ] @@ -6631,10 +6640,10 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: check command begin 2.36: --exec-id=1137-2618f7c4 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", + "P00 INFO: check command begin 2.37: --exec-id=977-a8bdb38d --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --stanza=demo", "P00 INFO: check repo1 configuration (primary)", "P00 INFO: check repo1 archive for WAL (primary)", - "P00 INFO: WAL segment 000000010000000000000001 successfully archived to '/var/lib/pgbackrest/archive/demo/10-1/0000000100000000/000000010000000000000001-78f9ebc180866af7d545351607e6c396464f0713.gz' on repo1", + "P00 INFO: WAL segment 000000010000000000000001 successfully archived to '/var/lib/pgbackrest/archive/demo/10-1/0000000100000000/000000010000000000000001-84287fe4b534f0cb6a7a693e206d5fddde3af7cb.gz' on repo1", "P00 INFO: check command end: completed successfully" ] } @@ -6691,16 +6700,16 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: backup command begin 2.36: --exec-id=1209-d88fd5f8 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo --start-fast", + "P00 INFO: backup command begin 2.37: --exec-id=1041-336b8131 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo --start-fast", "P00 WARN: no prior backup exists, incr backup has been changed to full", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", "P00 INFO: backup start archive = 000000010000000000000002, lsn = 0/2000028", - " [filtered 2 lines of output]", - "P00 INFO: check archive for segment(s) 000000010000000000000002:000000010000000000000002", - "P00 INFO: new backup label = 20211029-120144F", + " [filtered 3 lines of output]", + "P00 INFO: check archive for segment(s) 000000010000000000000002:000000010000000000000003", + "P00 INFO: new backup label = 20211231-194304F", "P00 INFO: full backup size = 22.5MB, file total = 949", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1209-d88fd5f8 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=1041-336b8131 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" ] } }, @@ -6718,7 +6727,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120144F" + "20211231-194304F" ] } }, @@ -6744,12 +6753,12 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 6 lines of output]", - "P00 INFO: check archive for segment(s) 000000010000000000000003:000000010000000000000003", - "P00 INFO: new backup label = 20211029-120144F_20211029-120148D", - "P00 INFO: diff backup size = 8KB, file total = 949", + " [filtered 7 lines of output]", + "P00 INFO: check archive for segment(s) 000000010000000000000004:000000010000000000000005", + "P00 INFO: new backup label = 20211231-194304F_20211231-194310D", + "P00 INFO: diff backup size = 8.8KB, file total = 949", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1282-04b54640 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=1096-e5efad67 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo" ] } }, @@ -6779,20 +6788,20 @@ " cipher: aes-256-cbc", "", " db (current)", - " wal archive min/max (10): 000000010000000000000001/000000010000000000000003", + " wal archive min/max (10): 000000010000000000000001/000000010000000000000005", "", - " full backup: 20211029-120144F", - " timestamp start/stop: 2021-10-29 12:01:44 / 2021-10-29 12:01:47", - " wal start/stop: 000000010000000000000002 / 000000010000000000000002", + " full backup: 20211231-194304F", + " timestamp start/stop: 2021-12-31 19:43:04 / 2021-12-31 19:43:08", + " wal start/stop: 000000010000000000000002 / 000000010000000000000003", " database size: 22.5MB, database backup size: 22.5MB", " repo1: backup set size: 2.7MB, backup size: 2.7MB", "", - " diff backup: 20211029-120144F_20211029-120148D", - " timestamp start/stop: 2021-10-29 12:01:48 / 2021-10-29 12:01:49", - " wal start/stop: 000000010000000000000003 / 000000010000000000000003", - " database size: 22.5MB, database backup size: 8.2KB", - " repo1: backup set size: 2.7MB, backup size: 496B", - " backup reference list: 20211029-120144F" + " diff backup: 20211231-194304F_20211231-194310D", + " timestamp start/stop: 2021-12-31 19:43:10 / 2021-12-31 19:43:12", + " wal start/stop: 000000010000000000000004 / 000000010000000000000005", + " database size: 22.5MB, database backup size: 8.8KB", + " repo1: backup set size: 2.7MB, backup size: 752B", + " backup reference list: 20211231-194304F" ] } }, @@ -6859,9 +6868,9 @@ "value" : { "output" : [ " [filtered 12 lines of output]", - "Oct 29 12:01:50 pg-primary systemd[1]: postgresql-10.service: Main process exited, code=exited, status=2/INVALIDARGUMENT", - "Oct 29 12:01:50 pg-primary systemd[1]: postgresql-10.service: Failed with result 'exit-code'.", - "Oct 29 12:01:50 pg-primary systemd[1]: Failed to start PostgreSQL 10 database server." + "Dec 31 19:43:15 pg-primary systemd[1]: postgresql-10.service: Main process exited, code=exited, status=2/INVALIDARGUMENT", + "Dec 31 19:43:15 pg-primary systemd[1]: postgresql-10.service: Failed with result 'exit-code'.", + "Dec 31 19:43:15 pg-primary systemd[1]: Failed to start PostgreSQL 10 database server." ] } }, @@ -7058,7 +7067,7 @@ "output" : [ " name | last_successful_backup | last_archived_wal ", "--------+------------------------+--------------------------", - " \"demo\" | 2021-10-29 12:01:49+00 | 000000010000000000000003", + " \"demo\" | 2021-12-31 19:43:12+00 | 000000010000000000000005", "(1 row)" ] } @@ -7104,7 +7113,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "archive retention on backup 20211029-120144F|remove archive" + "archive retention on backup 20211231-194304F|remove archive" ] }, "host" : "pg-primary", @@ -7115,10 +7124,10 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 957 lines of output]", + " [filtered 958 lines of output]", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1839-0cd4fa17 --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", - "P00 DETAIL: repo1: 10-1 archive retention on backup 20211029-120144F, start = 000000010000000000000002", + "P00 INFO: expire command begin 2.37: --exec-id=1555-052ad819 --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", + "P00 DETAIL: repo1: 10-1 archive retention on backup 20211231-194304F, start = 000000010000000000000002", "P00 INFO: repo1: 10-1 remove archive, start = 000000010000000000000001, stop = 000000010000000000000001", "P00 INFO: expire command end: completed successfully" ] @@ -7138,7 +7147,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120155F" + "20211231-194327F" ] } }, @@ -7153,7 +7162,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "expire full backup set 20211029-120144F|archive retention on backup 20211029-120155F|remove archive" + "expire full backup set 20211231-194304F|archive retention on backup 20211231-194327F|remove archive" ] }, "host" : "pg-primary", @@ -7164,13 +7173,13 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 8 lines of output]", + " [filtered 9 lines of output]", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=1914-8d3f4587 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", - "P00 INFO: repo1: expire full backup set 20211029-120144F, 20211029-120144F_20211029-120148D", - "P00 INFO: repo1: remove expired backup 20211029-120144F_20211029-120148D", - "P00 INFO: repo1: remove expired backup 20211029-120144F", - "P00 INFO: repo1: 10-1 remove archive, start = 0000000100000000, stop = 000000020000000000000004", + "P00 INFO: expire command begin 2.37: --exec-id=1610-9b83d72f --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-full=2 --stanza=demo", + "P00 INFO: repo1: expire full backup set 20211231-194304F, 20211231-194304F_20211231-194310D", + "P00 INFO: repo1: remove expired backup 20211231-194304F_20211231-194310D", + "P00 INFO: repo1: remove expired backup 20211231-194304F", + "P00 INFO: repo1: 10-1 remove archive, start = 0000000100000000, stop = 000000020000000000000006", "P00 INFO: expire command end: completed successfully" ] } @@ -7233,7 +7242,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120159F_20211029-120202D" + "20211231-194333F_20211231-194340D" ] } }, @@ -7261,7 +7270,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "expire diff backup set 20211029-120159F_20211029-120202D" + "expire diff backup set 20211231-194333F_20211231-194340D" ] }, "host" : "pg-primary", @@ -7272,12 +7281,12 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 9 lines of output]", + " [filtered 10 lines of output]", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=2101-7780704c --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-diff=1 --repo1-retention-full=2 --stanza=demo", - "P00 INFO: repo1: expire diff backup set 20211029-120159F_20211029-120202D, 20211029-120159F_20211029-120204I", - "P00 INFO: repo1: remove expired backup 20211029-120159F_20211029-120204I", - "P00 INFO: repo1: remove expired backup 20211029-120159F_20211029-120202D", + "P00 INFO: expire command begin 2.37: --exec-id=1761-d5f40afa --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-diff=1 --repo1-retention-full=2 --stanza=demo", + "P00 INFO: repo1: expire diff backup set 20211231-194333F_20211231-194340D, 20211231-194333F_20211231-194343I", + "P00 INFO: repo1: remove expired backup 20211231-194333F_20211231-194343I", + "P00 INFO: repo1: remove expired backup 20211231-194333F_20211231-194340D", "P00 INFO: expire command end: completed successfully" ] } @@ -7327,7 +7336,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120159F_20211029-120207D" + "20211231-194333F_20211231-194345D" ] } }, @@ -7368,11 +7377,11 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 5 lines of output]", - "P00 INFO: backup stop archive = 00000002000000000000000D, lsn = 0/D0000F8", - "P00 INFO: check archive for segment(s) 00000002000000000000000D:00000002000000000000000D", - "P00 INFO: new backup label = 20211029-120159F_20211029-120210D", - "P00 INFO: diff backup size = 9.2KB, file total = 949", + " [filtered 6 lines of output]", + "P00 INFO: backup stop archive = 000000020000000000000013, lsn = 0/13000050", + "P00 INFO: check archive for segment(s) 000000020000000000000012:000000020000000000000013", + "P00 INFO: new backup label = 20211231-194333F_20211231-194351D", + "P00 INFO: diff backup size = 10.6KB, file total = 949", "P00 INFO: backup command end: completed successfully", " [filtered 2 lines of output]" ] @@ -7392,7 +7401,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120159F_20211029-120210D" + "20211231-194333F_20211231-194351D" ] } }, @@ -7407,7 +7416,7 @@ "filter" : true, "filter-context" : 2, "list" : [ - "archive retention on backup 20211029-120159F_20211029-120207D|remove archive" + "archive retention on backup 20211231-194333F_20211231-194345D|remove archive" ] }, "host" : "pg-primary", @@ -7418,13 +7427,13 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: expire command begin 2.36: --exec-id=2333-931c5882 --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-archive=1 --repo1-retention-archive-type=diff --repo1-retention-diff=2 --repo1-retention-full=2 --stanza=demo", - "P00 DETAIL: repo1: 10-1 archive retention on backup 20211029-120155F, start = 000000020000000000000005, stop = 000000020000000000000005", - "P00 DETAIL: repo1: 10-1 archive retention on backup 20211029-120159F, start = 000000020000000000000006, stop = 000000020000000000000006", - "P00 DETAIL: repo1: 10-1 archive retention on backup 20211029-120159F_20211029-120207D, start = 00000002000000000000000A, stop = 00000002000000000000000A", - "P00 DETAIL: repo1: 10-1 archive retention on backup 20211029-120159F_20211029-120210D, start = 00000002000000000000000D", - "P00 INFO: repo1: 10-1 remove archive, start = 000000020000000000000007, stop = 000000020000000000000009", - "P00 INFO: repo1: 10-1 remove archive, start = 00000002000000000000000B, stop = 00000002000000000000000C", + "P00 INFO: expire command begin 2.37: --exec-id=1936-5402b1a3 --log-level-console=detail --log-level-stderr=off --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo1-retention-archive=1 --repo1-retention-archive-type=diff --repo1-retention-diff=2 --repo1-retention-full=2 --stanza=demo", + "P00 DETAIL: repo1: 10-1 archive retention on backup 20211231-194327F, start = 000000020000000000000007, stop = 000000020000000000000007", + "P00 DETAIL: repo1: 10-1 archive retention on backup 20211231-194333F, start = 000000020000000000000008, stop = 000000020000000000000009", + "P00 DETAIL: repo1: 10-1 archive retention on backup 20211231-194333F_20211231-194345D, start = 00000002000000000000000E, stop = 00000002000000000000000F", + "P00 DETAIL: repo1: 10-1 archive retention on backup 20211231-194333F_20211231-194351D, start = 000000020000000000000012", + "P00 INFO: repo1: 10-1 remove archive, start = 00000002000000000000000A, stop = 00000002000000000000000D", + "P00 INFO: repo1: 10-1 remove archive, start = 000000020000000000000010, stop = 000000020000000000000011", "P00 INFO: expire command end: completed successfully" ] } @@ -7470,7 +7479,7 @@ "P00 INFO: remove invalid files/links/paths from '/var/lib/pgsql/10/data'", "P00 DETAIL: remove invalid file '/var/lib/pgsql/10/data/backup_label.old'", "P00 DETAIL: remove invalid file '/var/lib/pgsql/10/data/base/13017/pg_internal.init'", - " [filtered 998 lines of output]" + " [filtered 1000 lines of output]" ] } }, @@ -7637,7 +7646,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120159F_20211029-120216I" + "20211231-194333F_20211231-194405I" ] } }, @@ -7646,7 +7655,7 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo \\", - " --set=20211029-120159F_20211029-120216I info" + " --set=20211231-194333F_20211231-194405I info" ], "highlight" : { "filter" : true, @@ -7664,8 +7673,8 @@ "value" : { "output" : [ " [filtered 11 lines of output]", - " repo1: backup set size: 4.4MB, backup size: 1.8MB", - " backup reference list: 20211029-120159F, 20211029-120159F_20211029-120210D", + " repo1: backup set size: 4.5MB, backup size: 1.8MB", + " backup reference list: 20211231-194333F, 20211231-194333F_20211231-194351D", " database list: postgres (13017), test1 (24576), test2 (24577)" ] } @@ -7909,7 +7918,7 @@ "type" : "exe", "value" : { "output" : [ - "2021-10-29 12:02:28.38486+00" + "2021-12-31 19:44:27.079264+00" ] } }, @@ -7975,7 +7984,7 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo --delta \\", - " --type=time \"--target=2021-10-29 12:02:28.38486+00\" \\", + " --type=time \"--target=2021-12-31 19:44:27.079264+00\" \\", " --target-action=promote restore" ], "host" : "pg-primary", @@ -8019,9 +8028,9 @@ "type" : "exe", "value" : { "output" : [ - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:02:30", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:44:31", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", - "recovery_target_time = '2021-10-29 12:02:28.38486+00'", + "recovery_target_time = '2021-12-31 19:44:27.079264+00'", "recovery_target_action = 'promote'" ] } @@ -8101,16 +8110,16 @@ "type" : "exe", "value" : { "output" : [ - "LOG: database system was interrupted; last known up at 2021-10-29 12:02:24 UTC", - "LOG: starting point-in-time recovery to 2021-10-29 12:02:28.38486+00", + "LOG: database system was interrupted; last known up at 2021-12-31 19:44:21 UTC", + "LOG: starting point-in-time recovery to 2021-12-31 19:44:27.079264+00", "LOG: restored log file \"00000004.history\" from archive", - "LOG: restored log file \"000000040000000000000010\" from archive", + "LOG: restored log file \"000000040000000000000016\" from archive", " [filtered 2 lines of output]", "LOG: database system is ready to accept read only connections", - "LOG: restored log file \"000000040000000000000011\" from archive", - "LOG: recovery stopping before commit of transaction 564, time 2021-10-29 12:02:29.695775+00", - "LOG: redo done at 0/11020858", - "LOG: last completed transaction was at log time 2021-10-29 12:02:26.985761+00", + "LOG: restored log file \"000000040000000000000017\" from archive", + "LOG: recovery stopping before commit of transaction 564, time 2021-12-31 19:44:29.264483+00", + "LOG: redo done at 0/17021810", + "LOG: last completed transaction was at log time 2021-12-31 19:44:24.848796+00", "LOG: selected new timeline ID: 5", "LOG: archive recovery complete", "LOG: database system is ready to accept connections" @@ -8175,7 +8184,7 @@ "type" : "exe", "value" : { "output" : [ - "20211029-120159F_20211029-120234I" + "20211231-194333F_20211231-194438I" ] } }, @@ -8189,7 +8198,7 @@ "filter" : false, "filter-context" : 2, "list" : [ - "20211029-120159F_20211029-120234I" + "20211231-194333F_20211231-194438I" ] }, "host" : "pg-primary", @@ -8205,47 +8214,47 @@ " cipher: aes-256-cbc", "", " db (current)", - " wal archive min/max (10): 000000020000000000000005/000000050000000000000012", + " wal archive min/max (10): 000000020000000000000007/000000050000000000000018", "", - " full backup: 20211029-120155F", - " timestamp start/stop: 2021-10-29 12:01:55 / 2021-10-29 12:01:58", - " wal start/stop: 000000020000000000000005 / 000000020000000000000005", + " full backup: 20211231-194327F", + " timestamp start/stop: 2021-12-31 19:43:27 / 2021-12-31 19:43:31", + " wal start/stop: 000000020000000000000007 / 000000020000000000000007", " database size: 22.5MB, database backup size: 22.5MB", " repo1: backup set size: 2.7MB, backup size: 2.7MB", "", - " full backup: 20211029-120159F", - " timestamp start/stop: 2021-10-29 12:01:59 / 2021-10-29 12:02:01", - " wal start/stop: 000000020000000000000006 / 000000020000000000000006", + " full backup: 20211231-194333F", + " timestamp start/stop: 2021-12-31 19:43:33 / 2021-12-31 19:43:37", + " wal start/stop: 000000020000000000000008 / 000000020000000000000009", " database size: 22.5MB, database backup size: 22.5MB", " repo1: backup set size: 2.7MB, backup size: 2.7MB", "", - " diff backup: 20211029-120159F_20211029-120210D", - " timestamp start/stop: 2021-10-29 12:02:10 / 2021-10-29 12:02:11", - " wal start/stop: 00000002000000000000000D / 00000002000000000000000D", - " database size: 22.5MB, database backup size: 9.5KB", - " repo1: backup set size: 2.7MB, backup size: 928B", - " backup reference list: 20211029-120159F", - "", - " incr backup: 20211029-120159F_20211029-120216I", - " timestamp start/stop: 2021-10-29 12:02:16 / 2021-10-29 12:02:18", - " wal start/stop: 00000003000000000000000F / 00000003000000000000000F", + " diff backup: 20211231-194333F_20211231-194351D", + " timestamp start/stop: 2021-12-31 19:43:51 / 2021-12-31 19:43:53", + " wal start/stop: 000000020000000000000012 / 000000020000000000000013", + " database size: 22.5MB, database backup size: 10.6KB", + " repo1: backup set size: 2.7MB, backup size: 992B", + " backup reference list: 20211231-194333F", + "", + " incr backup: 20211231-194333F_20211231-194405I", + " timestamp start/stop: 2021-12-31 19:44:05 / 2021-12-31 19:44:08", + " wal start/stop: 000000030000000000000015 / 000000030000000000000015", " database size: 37.2MB, database backup size: 15.2MB", - " repo1: backup set size: 4.4MB, backup size: 1.8MB", - " backup reference list: 20211029-120159F, 20211029-120159F_20211029-120210D", + " repo1: backup set size: 4.5MB, backup size: 1.8MB", + " backup reference list: 20211231-194333F, 20211231-194333F_20211231-194351D", "", - " diff backup: 20211029-120159F_20211029-120224D", - " timestamp start/stop: 2021-10-29 12:02:24 / 2021-10-29 12:02:26", - " wal start/stop: 000000040000000000000010 / 000000040000000000000010", + " diff backup: 20211231-194333F_20211231-194421D", + " timestamp start/stop: 2021-12-31 19:44:21 / 2021-12-31 19:44:24", + " wal start/stop: 000000040000000000000016 / 000000040000000000000016", " database size: 29.9MB, database backup size: 7.8MB", - " repo1: backup set size: 3.6MB, backup size: 951.8KB", - " backup reference list: 20211029-120159F", + " repo1: backup set size: 3.6MB, backup size: 952.3KB", + " backup reference list: 20211231-194333F", "", - " incr backup: 20211029-120159F_20211029-120234I", - " timestamp start/stop: 2021-10-29 12:02:34 / 2021-10-29 12:02:36", - " wal start/stop: 000000050000000000000012 / 000000050000000000000012", + " incr backup: 20211231-194333F_20211231-194438I", + " timestamp start/stop: 2021-12-31 19:44:38 / 2021-12-31 19:44:40", + " wal start/stop: 000000050000000000000018 / 000000050000000000000018", " database size: 29.9MB, database backup size: 2.1MB", - " repo1: backup set size: 3.6MB, backup size: 218.4KB", - " backup reference list: 20211029-120159F, 20211029-120159F_20211029-120224D" + " repo1: backup set size: 3.6MB, backup size: 218.7KB", + " backup reference list: 20211231-194333F, 20211231-194333F_20211231-194421D" ] } }, @@ -8267,8 +8276,8 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo --delta \\", - " --set=20211029-120159F_20211029-120234I \\", - " --type=time \"--target=2021-10-29 12:02:28.38486+00\" --target-action=promote restore" + " --set=20211231-194333F_20211231-194438I \\", + " --type=time \"--target=2021-12-31 19:44:27.079264+00\" --target-action=promote restore" ], "host" : "pg-primary", "load-env" : true, @@ -8365,14 +8374,14 @@ "type" : "exe", "value" : { "output" : [ - "LOG: database system was interrupted; last known up at 2021-10-29 12:02:34 UTC", - "LOG: starting point-in-time recovery to 2021-10-29 12:02:28.38486+00", + "LOG: database system was interrupted; last known up at 2021-12-31 19:44:38 UTC", + "LOG: starting point-in-time recovery to 2021-12-31 19:44:27.079264+00", "LOG: restored log file \"00000005.history\" from archive", - "LOG: restored log file \"000000050000000000000012\" from archive", - "LOG: redo starts at 0/12000028", - "LOG: consistent recovery state reached at 0/120000F8", + "LOG: restored log file \"000000050000000000000018\" from archive", + "LOG: redo starts at 0/18000028", + "LOG: consistent recovery state reached at 0/180000F8", "LOG: database system is ready to accept read only connections", - "LOG: redo done at 0/120000F8", + "LOG: redo done at 0/180000F8", " [filtered 6 lines of output]" ] } @@ -8395,7 +8404,7 @@ "bash-wrap" : true, "cmd" : [ "sudo -u postgres pgbackrest --stanza=demo --delta \\", - " --type=time \"--target=2021-10-29 12:02:28.38486+00\" \\", + " --type=time \"--target=2021-12-31 19:44:27.079264+00\" \\", " --target-action=promote restore" ], "host" : "pg-primary", @@ -8493,16 +8502,16 @@ "type" : "exe", "value" : { "output" : [ - "LOG: database system was interrupted; last known up at 2021-10-29 12:02:24 UTC", - "LOG: starting point-in-time recovery to 2021-10-29 12:02:28.38486+00", + "LOG: database system was interrupted; last known up at 2021-12-31 19:44:21 UTC", + "LOG: starting point-in-time recovery to 2021-12-31 19:44:27.079264+00", "LOG: restored log file \"00000004.history\" from archive", - "LOG: restored log file \"000000040000000000000010\" from archive", + "LOG: restored log file \"000000040000000000000016\" from archive", " [filtered 2 lines of output]", "LOG: database system is ready to accept read only connections", - "LOG: restored log file \"000000040000000000000011\" from archive", - "LOG: recovery stopping before commit of transaction 564, time 2021-10-29 12:02:29.695775+00", - "LOG: redo done at 0/11020858", - "LOG: last completed transaction was at log time 2021-10-29 12:02:26.985761+00", + "LOG: restored log file \"000000040000000000000017\" from archive", + "LOG: recovery stopping before commit of transaction 564, time 2021-12-31 19:44:29.264483+00", + "LOG: redo done at 0/17021810", + "LOG: last completed transaction was at log time 2021-12-31 19:44:24.848796+00", "LOG: restored log file \"00000005.history\" from archive", "LOG: restored log file \"00000006.history\" from archive", " [filtered 3 lines of output]" @@ -8641,16 +8650,16 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: backup command begin 2.36: --exec-id=4271-908fb51a --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=4 --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo --start-fast", + "P00 INFO: backup command begin 2.37: --exec-id=3515-0d0165d7 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=4 --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo --start-fast", "P00 WARN: no prior backup exists, incr backup has been changed to full", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", - "P00 INFO: backup start archive = 000000070000000000000012, lsn = 0/12000028", - " [filtered 2 lines of output]", - "P00 INFO: check archive for segment(s) 000000070000000000000012:000000070000000000000012", - "P00 INFO: new backup label = 20211029-120246F", + "P00 INFO: backup start archive = 000000070000000000000018, lsn = 0/18000028", + " [filtered 3 lines of output]", + "P00 INFO: check archive for segment(s) 000000070000000000000018:000000070000000000000018", + "P00 INFO: new backup label = 20211231-194501F", "P00 INFO: full backup size = 29.9MB, file total = 1246", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=4271-908fb51a --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=3515-0d0165d7 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=2 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo2-type=azure --stanza=demo" ] } }, @@ -8800,16 +8809,16 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: backup command begin 2.36: --exec-id=4447-40a22558 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=4 --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo --start-fast", + "P00 INFO: backup command begin 2.37: --exec-id=3655-556e7ff2 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=4 --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo --start-fast", "P00 WARN: no prior backup exists, incr backup has been changed to full", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", - "P00 INFO: backup start archive = 000000070000000000000013, lsn = 0/13000028", - " [filtered 2 lines of output]", - "P00 INFO: check archive for segment(s) 000000070000000000000013:000000070000000000000013", - "P00 INFO: new backup label = 20211029-120253F", + "P00 INFO: backup start archive = 000000070000000000000019, lsn = 0/19000028", + " [filtered 3 lines of output]", + "P00 INFO: check archive for segment(s) 000000070000000000000019:00000007000000000000001A", + "P00 INFO: new backup label = 20211231-194519F", "P00 INFO: full backup size = 29.9MB, file total = 1246", "P00 INFO: backup command end: completed successfully", - "P00 INFO: expire command begin 2.36: --exec-id=4447-40a22558 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo" + "P00 INFO: expire command begin 2.37: --exec-id=3655-556e7ff2 --log-level-console=info --log-level-stderr=off --no-log-timestamp --repo=3 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo1-retention-diff=2 --repo1-retention-full=2 --repo2-retention-full=4 --repo3-retention-full=4 --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --stanza=demo" ] } }, @@ -8909,7 +8918,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stop command begin 2.36: --exec-id=4565-85a19a66 --log-level-console=info --log-level-stderr=off --no-log-timestamp --stanza=demo", + "P00 INFO: stop command begin 2.37: --exec-id=3749-8d7b18b8 --log-level-console=info --log-level-stderr=off --no-log-timestamp --stanza=demo", "P00 INFO: stop command end: completed successfully" ] } @@ -8936,7 +8945,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stanza-delete command begin 2.36: --exec-id=4600-ce656d1f --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo=1 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo4-gcs-bucket=demo-bucket --repo4-gcs-key= --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo4-path=/demo-repo --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --repo4-type=gcs --stanza=demo", + "P00 INFO: stanza-delete command begin 2.37: --exec-id=3773-1ad02626 --log-level-console=info --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo=1 --repo2-azure-account= --repo2-azure-container=demo-container --repo2-azure-key= --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo4-gcs-bucket=demo-bucket --repo4-gcs-key= --repo1-path=/var/lib/pgbackrest --repo2-path=/demo-repo --repo3-path=/demo-repo --repo4-path=/demo-repo --repo3-s3-bucket=demo-bucket --repo3-s3-endpoint=s3.us-east-1.amazonaws.com --repo3-s3-key= --repo3-s3-key-secret= --repo3-s3-region=us-east-1 --repo2-type=azure --repo3-type=s3 --repo4-type=gcs --stanza=demo", "P00 INFO: stanza-delete command end: completed successfully" ] } @@ -9012,7 +9021,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo scp build:/build/pgbackrest-release-2.36/src/pgbackrest /usr/bin" + "sudo scp build:/build/pgbackrest-release-2.37/src/pgbackrest /usr/bin" ], "cmd-extra" : "2>&1", "host" : "repository", @@ -9167,120 +9176,6 @@ }, { "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u pgbackrest mkdir -m 750 /home/pgbackrest/.ssh" - ], - "host" : "repository", - "load-env" : true, - "output" : false, - "run-as-user" : null - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u pgbackrest ssh-keygen -f /home/pgbackrest/.ssh/id_rsa \\", - " -t rsa -b 4096 -N \"\"" - ], - "host" : "repository", - "load-env" : true, - "output" : false, - "run-as-user" : null - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u postgres mkdir -m 750 -p /var/lib/pgsql/.ssh" - ], - "host" : "pg-primary", - "load-env" : true, - "output" : false, - "run-as-user" : null - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u postgres ssh-keygen -f /var/lib/pgsql/.ssh/id_rsa \\", - " -t rsa -b 4096 -N \"\"" - ], - "host" : "pg-primary", - "load-env" : true, - "output" : false, - "run-as-user" : null - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "(echo -n 'no-agent-forwarding,no-X11-forwarding,no-port-forwarding,' && \\", - " echo -n 'command=\"/usr/bin/pgbackrest ${SSH_ORIGINAL_COMMAND#* }\" ' && \\", - " sudo ssh root@pg-primary cat /var/lib/pgsql/.ssh/id_rsa.pub) | \\", - " sudo -u pgbackrest tee -a /home/pgbackrest/.ssh/authorized_keys" - ], - "host" : "repository", - "load-env" : true, - "output" : false, - "run-as-user" : "root" - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "(echo -n 'no-agent-forwarding,no-X11-forwarding,no-port-forwarding,' && \\", - " echo -n 'command=\"/usr/bin/pgbackrest ${SSH_ORIGINAL_COMMAND#* }\" ' && \\", - " sudo ssh root@repository cat /home/pgbackrest/.ssh/id_rsa.pub) | \\", - " sudo -u postgres tee -a /var/lib/pgsql/.ssh/authorized_keys" - ], - "host" : "pg-primary", - "load-env" : true, - "output" : false, - "run-as-user" : "root" - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u pgbackrest ssh postgres@pg-primary" - ], - "cmd-extra" : "-o StrictHostKeyChecking=no", - "host" : "repository", - "load-env" : true, - "output" : false, - "run-as-user" : null - }, - "type" : "exe" - }, - { - "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u postgres ssh pgbackrest@repository" - ], - "cmd-extra" : "-o StrictHostKeyChecking=no", - "host" : "pg-primary", - "load-env" : true, - "output" : false, - "run-as-user" : null - }, - "type" : "exe" - }, - { - "key" : { "file" : "/etc/pgbackrest/pgbackrest.conf", "host" : "repository", "option" : { @@ -9308,6 +9203,18 @@ "pg1-host" : { "value" : "pg-primary" }, + "pg1-host-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "pg1-host-cert-file" : { + "value" : "/etc/pgbackrest/cert/client.crt" + }, + "pg1-host-key-file" : { + "value" : "/etc/pgbackrest/cert/client.key" + }, + "pg1-host-type" : { + "value" : "tls" + }, "pg1-path" : { "value" : "/var/lib/pgsql/10/data" } @@ -9324,6 +9231,21 @@ }, "start-fast" : { "value" : "y" + }, + "tls-server-address" : { + "value" : "*" + }, + "tls-server-auth" : { + "value" : "pgbackrest-client=demo" + }, + "tls-server-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "tls-server-cert-file" : { + "value" : "/etc/pgbackrest/cert/server.crt" + }, + "tls-server-key-file" : { + "value" : "/etc/pgbackrest/cert/server.key" } } } @@ -9333,12 +9255,21 @@ "config" : [ "[demo]", "pg1-host=pg-primary", + "pg1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg1-host-key-file=/etc/pgbackrest/cert/client.key", + "pg1-host-type=tls", "pg1-path=/var/lib/pgsql/10/data", "", "[global]", "repo1-path=/var/lib/pgbackrest", "repo1-retention-full=2", - "start-fast=y" + "start-fast=y", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" ] } }, @@ -9350,6 +9281,18 @@ "demo" : { "pg1-path" : { "value" : "/var/lib/pgsql/10/data" + }, + "repo1-host-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "repo1-host-cert-file" : { + "value" : "/etc/pgbackrest/cert/client.crt" + }, + "repo1-host-key-file" : { + "value" : "/etc/pgbackrest/cert/client.key" + }, + "repo1-host-type" : { + "value" : "tls" } }, "global" : { @@ -9364,6 +9307,21 @@ }, "repo1-host" : { "value" : "repository" + }, + "tls-server-address" : { + "value" : "*" + }, + "tls-server-auth" : { + "value" : "pgbackrest-client=demo" + }, + "tls-server-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "tls-server-cert-file" : { + "value" : "/etc/pgbackrest/cert/server.crt" + }, + "tls-server-key-file" : { + "value" : "/etc/pgbackrest/cert/server.key" } } }, @@ -9374,10 +9332,234 @@ "config" : [ "[demo]", "pg1-path=/var/lib/pgsql/10/data", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", "", "[global]", "log-level-file=detail", - "repo1-host=repository" + "repo1-host=repository", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" + ] + } + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "mkdir -p -m 770 /etc/pgbackrest/cert && \\", + " cp /pgbackrest/doc/resource/fake-cert/ca.crt \\", + " /etc/pgbackrest/cert/ca.crt && \\", + " \\", + " openssl genrsa -out /etc/pgbackrest/cert/server.key 2048 2>&1 && \\", + " chmod 600 /etc/pgbackrest/cert/server.key && \\", + " openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/server.csr \\", + " -key /etc/pgbackrest/cert/server.key -subj \"/CN=repository\" 2>&1 && \\", + " openssl x509 -req -in /etc/pgbackrest/cert/server.csr \\", + " -CA /etc/pgbackrest/cert/ca.crt \\", + " -CAkey /pgbackrest/doc/resource/fake-cert/ca.key -CAcreateserial \\", + " -out /etc/pgbackrest/cert/server.crt -days 9 2>&1 && \\", + " \\", + " openssl genrsa -out /etc/pgbackrest/cert/client.key 2048 2>&1 && \\", + " chmod 600 /etc/pgbackrest/cert/client.key && \\", + " openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/client.csr \\", + " -key /etc/pgbackrest/cert/client.key -subj \"/CN=pgbackrest-client\" 2>&1 && \\", + " openssl x509 -req -in /etc/pgbackrest/cert/client.csr \\", + " -CA /etc/pgbackrest/cert/ca.crt \\", + " -CAkey /pgbackrest/doc/resource/fake-cert/ca.key -CAcreateserial \\", + " -out /etc/pgbackrest/cert/client.crt -days 9 2>&1 && \\", + " \\", + " chown -R pgbackrest /etc/pgbackrest/cert" + ], + "host" : "repository", + "load-env" : true, + "output" : false, + "run-as-user" : "root" + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "echo '[Unit]' | tee /etc/systemd/system/pgbackrest.service && \\", + " echo 'Description=pgBackRest Server' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'After=network.target' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'StartLimitIntervalSec=0' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '[Service]' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'Type=simple' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'Restart=always' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'RestartSec=1' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'User=pgbackrest' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'ExecStart=/usr/bin/pgbackrest server' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'ExecReload=kill -HUP $MAINPID' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '[Install]' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'WantedBy=multi-user.target' | tee -a /etc/systemd/system/pgbackrest.service" + ], + "host" : "repository", + "load-env" : true, + "output" : false, + "run-as-user" : "root" + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "sudo cat /etc/systemd/system/pgbackrest.service" + ], + "host" : "repository", + "load-env" : true, + "output" : true, + "run-as-user" : null + }, + "type" : "exe", + "value" : { + "output" : [ + "[Unit]", + "Description=pgBackRest Server", + "After=network.target", + "StartLimitIntervalSec=0", + "", + "[Service]", + "Type=simple", + "Restart=always", + "RestartSec=1", + "User=pgbackrest", + "ExecStart=/usr/bin/pgbackrest server", + "ExecReload=kill -HUP $MAINPID", + "", + "[Install]", + "WantedBy=multi-user.target" + ] + } + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "sudo systemctl enable pgbackrest" + ], + "cmd-extra" : "2>&1", + "host" : "repository", + "load-env" : true, + "output" : false, + "run-as-user" : null + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "sudo systemctl start pgbackrest" + ], + "host" : "repository", + "load-env" : true, + "output" : false, + "run-as-user" : null + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "mkdir -p -m 770 /etc/pgbackrest/cert && \\", + " cp /pgbackrest/doc/resource/fake-cert/ca.crt \\", + " /etc/pgbackrest/cert/ca.crt && \\", + " \\", + " openssl genrsa -out /etc/pgbackrest/cert/server.key 2048 2>&1 && \\", + " chmod 600 /etc/pgbackrest/cert/server.key && \\", + " openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/server.csr \\", + " -key /etc/pgbackrest/cert/server.key -subj \"/CN=pg-primary\" 2>&1 && \\", + " openssl x509 -req -in /etc/pgbackrest/cert/server.csr \\", + " -CA /etc/pgbackrest/cert/ca.crt \\", + " -CAkey /pgbackrest/doc/resource/fake-cert/ca.key -CAcreateserial \\", + " -out /etc/pgbackrest/cert/server.crt -days 9 2>&1 && \\", + " \\", + " openssl genrsa -out /etc/pgbackrest/cert/client.key 2048 2>&1 && \\", + " chmod 600 /etc/pgbackrest/cert/client.key && \\", + " openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/client.csr \\", + " -key /etc/pgbackrest/cert/client.key -subj \"/CN=pgbackrest-client\" 2>&1 && \\", + " openssl x509 -req -in /etc/pgbackrest/cert/client.csr \\", + " -CA /etc/pgbackrest/cert/ca.crt \\", + " -CAkey /pgbackrest/doc/resource/fake-cert/ca.key -CAcreateserial \\", + " -out /etc/pgbackrest/cert/client.crt -days 9 2>&1 && \\", + " \\", + " chown -R postgres /etc/pgbackrest/cert" + ], + "host" : "pg-primary", + "load-env" : true, + "output" : false, + "run-as-user" : "root" + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "echo '[Unit]' | tee /etc/systemd/system/pgbackrest.service && \\", + " echo 'Description=pgBackRest Server' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'After=network.target' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'StartLimitIntervalSec=0' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '[Service]' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'Type=simple' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'Restart=always' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'RestartSec=1' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'User=postgres' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'ExecStart=/usr/bin/pgbackrest server' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'ExecReload=kill -HUP $MAINPID' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '[Install]' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'WantedBy=multi-user.target' | tee -a /etc/systemd/system/pgbackrest.service" + ], + "host" : "pg-primary", + "load-env" : true, + "output" : false, + "run-as-user" : "root" + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "sudo cat /etc/systemd/system/pgbackrest.service" + ], + "host" : "pg-primary", + "load-env" : true, + "output" : true, + "run-as-user" : null + }, + "type" : "exe", + "value" : { + "output" : [ + "[Unit]", + "Description=pgBackRest Server", + "After=network.target", + "StartLimitIntervalSec=0", + "", + "[Service]", + "Type=simple", + "Restart=always", + "RestartSec=1", + "User=postgres", + "ExecStart=/usr/bin/pgbackrest server", + "ExecReload=kill -HUP $MAINPID", + "", + "[Install]", + "WantedBy=multi-user.target" ] } }, @@ -9385,6 +9567,33 @@ "key" : { "bash-wrap" : true, "cmd" : [ + "sudo systemctl enable pgbackrest" + ], + "cmd-extra" : "2>&1", + "host" : "pg-primary", + "load-env" : true, + "output" : false, + "run-as-user" : null + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ + "sudo systemctl start pgbackrest" + ], + "host" : "pg-primary", + "load-env" : true, + "output" : false, + "run-as-user" : null + }, + "type" : "exe" + }, + { + "key" : { + "bash-wrap" : true, + "cmd" : [ "sudo -u pgbackrest pgbackrest --stanza=demo stanza-create" ], "host" : "repository", @@ -9520,13 +9729,22 @@ "config" : [ "[demo]", "pg1-host=pg-primary", + "pg1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg1-host-key-file=/etc/pgbackrest/cert/client.key", + "pg1-host-type=tls", "pg1-path=/var/lib/pgsql/10/data", "", "[global]", "process-max=3", "repo1-path=/var/lib/pgbackrest", "repo1-retention-full=2", - "start-fast=y" + "start-fast=y", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" ] } }, @@ -9569,17 +9787,17 @@ " cipher: none", "", " db (current)", - " wal archive min/max (10): 000000080000000000000018/000000080000000000000019", + " wal archive min/max (10): 000000080000000000000021/000000080000000000000023", "", - " full backup: 20211029-120319F", - " timestamp start/stop: 2021-10-29 12:03:19 / 2021-10-29 12:03:21", - " wal start/stop: 000000080000000000000018 / 000000080000000000000018", + " full backup: 20211231-194712F", + " timestamp start/stop: 2021-12-31 19:47:12 / 2021-12-31 19:47:17", + " wal start/stop: 000000080000000000000021 / 000000080000000000000021", " database size: 29.9MB, database backup size: 29.9MB", " repo1: backup set size: 3.5MB, backup size: 3.5MB", "", - " full backup: 20211029-120323F", - " timestamp start/stop: 2021-10-29 12:03:23 / 2021-10-29 12:03:26", - " wal start/stop: 000000080000000000000019 / 000000080000000000000019", + " full backup: 20211231-194720F", + " timestamp start/stop: 2021-12-31 19:47:20 / 2021-12-31 19:47:25", + " wal start/stop: 000000080000000000000022 / 000000080000000000000023", " database size: 29.9MB, database backup size: 29.9MB", " repo1: backup set size: 3.5MB, backup size: 3.5MB" ] @@ -9620,7 +9838,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 WARN: unable to check pg-1: [StopError] raised from remote-0 ssh protocol on 'pg-primary': stop file exists for all stanzas", + "P00 WARN: unable to check pg-1: [StopError] raised from remote-0 tls protocol on 'pg-primary': stop file exists for all stanzas", "P00 ERROR: [056]: unable to find primary cluster - cannot proceed" ] } @@ -9691,7 +9909,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 WARN: unable to check pg-1: [StopError] raised from remote-0 ssh protocol on 'pg-primary': stop file exists for stanza demo", + "P00 WARN: unable to check pg-1: [StopError] raised from remote-0 tls protocol on 'pg-primary': stop file exists for stanza demo", "P00 ERROR: [056]: unable to find primary cluster - cannot proceed" ] } @@ -9741,7 +9959,7 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo scp build:/build/pgbackrest-release-2.36/src/pgbackrest /usr/bin" + "sudo scp build:/build/pgbackrest-release-2.37/src/pgbackrest /usr/bin" ], "cmd-extra" : "2>&1", "host" : "pg-standby", @@ -9857,28 +10075,110 @@ }, { "key" : { - "bash-wrap" : true, - "cmd" : [ - "sudo -u postgres mkdir -m 750 -p /var/lib/pgsql/.ssh" - ], + "file" : "/etc/pgbackrest/pgbackrest.conf", "host" : "pg-standby", - "load-env" : true, - "output" : false, - "run-as-user" : null + "option" : { + "demo" : { + "pg1-path" : { + "value" : "/var/lib/pgsql/10/data" + }, + "repo1-host-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "repo1-host-cert-file" : { + "value" : "/etc/pgbackrest/cert/client.crt" + }, + "repo1-host-key-file" : { + "value" : "/etc/pgbackrest/cert/client.key" + }, + "repo1-host-type" : { + "value" : "tls" + } + }, + "global" : { + "log-level-file" : { + "value" : "detail" + }, + "log-level-stderr" : { + "value" : "off" + }, + "log-timestamp" : { + "value" : "n" + }, + "repo1-host" : { + "value" : "repository" + }, + "tls-server-address" : { + "value" : "*" + }, + "tls-server-auth" : { + "value" : "pgbackrest-client=demo" + }, + "tls-server-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "tls-server-cert-file" : { + "value" : "/etc/pgbackrest/cert/server.crt" + }, + "tls-server-key-file" : { + "value" : "/etc/pgbackrest/cert/server.key" + } + } + } }, - "type" : "exe" + "type" : "cfg-pgbackrest", + "value" : { + "config" : [ + "[demo]", + "pg1-path=/var/lib/pgsql/10/data", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", + "", + "[global]", + "log-level-file=detail", + "repo1-host=repository", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" + ] + } }, { "key" : { "bash-wrap" : true, "cmd" : [ - "sudo -u postgres ssh-keygen -f /var/lib/pgsql/.ssh/id_rsa \\", - " -t rsa -b 4096 -N \"\"" + "mkdir -p -m 770 /etc/pgbackrest/cert && \\", + " cp /pgbackrest/doc/resource/fake-cert/ca.crt \\", + " /etc/pgbackrest/cert/ca.crt && \\", + " \\", + " openssl genrsa -out /etc/pgbackrest/cert/server.key 2048 2>&1 && \\", + " chmod 600 /etc/pgbackrest/cert/server.key && \\", + " openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/server.csr \\", + " -key /etc/pgbackrest/cert/server.key -subj \"/CN=pg-standby\" 2>&1 && \\", + " openssl x509 -req -in /etc/pgbackrest/cert/server.csr \\", + " -CA /etc/pgbackrest/cert/ca.crt \\", + " -CAkey /pgbackrest/doc/resource/fake-cert/ca.key -CAcreateserial \\", + " -out /etc/pgbackrest/cert/server.crt -days 9 2>&1 && \\", + " \\", + " openssl genrsa -out /etc/pgbackrest/cert/client.key 2048 2>&1 && \\", + " chmod 600 /etc/pgbackrest/cert/client.key && \\", + " openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/client.csr \\", + " -key /etc/pgbackrest/cert/client.key -subj \"/CN=pgbackrest-client\" 2>&1 && \\", + " openssl x509 -req -in /etc/pgbackrest/cert/client.csr \\", + " -CA /etc/pgbackrest/cert/ca.crt \\", + " -CAkey /pgbackrest/doc/resource/fake-cert/ca.key -CAcreateserial \\", + " -out /etc/pgbackrest/cert/client.crt -days 9 2>&1 && \\", + " \\", + " chown -R postgres /etc/pgbackrest/cert" ], "host" : "pg-standby", "load-env" : true, "output" : false, - "run-as-user" : null + "run-as-user" : "root" }, "type" : "exe" }, @@ -9886,12 +10186,23 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "(echo -n 'no-agent-forwarding,no-X11-forwarding,no-port-forwarding,' && \\", - " echo -n 'command=\"/usr/bin/pgbackrest ${SSH_ORIGINAL_COMMAND#* }\" ' && \\", - " sudo ssh root@pg-standby cat /var/lib/pgsql/.ssh/id_rsa.pub) | \\", - " sudo -u pgbackrest tee -a /home/pgbackrest/.ssh/authorized_keys" + "echo '[Unit]' | tee /etc/systemd/system/pgbackrest.service && \\", + " echo 'Description=pgBackRest Server' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'After=network.target' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'StartLimitIntervalSec=0' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '[Service]' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'Type=simple' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'Restart=always' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'RestartSec=1' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'User=postgres' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'ExecStart=/usr/bin/pgbackrest server' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'ExecReload=kill -HUP $MAINPID' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo '[Install]' | tee -a /etc/systemd/system/pgbackrest.service && \\", + " echo 'WantedBy=multi-user.target' | tee -a /etc/systemd/system/pgbackrest.service" ], - "host" : "repository", + "host" : "pg-standby", "load-env" : true, "output" : false, "run-as-user" : "root" @@ -9902,26 +10213,42 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "(echo -n 'no-agent-forwarding,no-X11-forwarding,no-port-forwarding,' && \\", - " echo -n 'command=\"/usr/bin/pgbackrest ${SSH_ORIGINAL_COMMAND#* }\" ' && \\", - " sudo ssh root@repository cat /home/pgbackrest/.ssh/id_rsa.pub) | \\", - " sudo -u postgres tee -a /var/lib/pgsql/.ssh/authorized_keys" + "sudo cat /etc/systemd/system/pgbackrest.service" ], "host" : "pg-standby", "load-env" : true, - "output" : false, - "run-as-user" : "root" + "output" : true, + "run-as-user" : null }, - "type" : "exe" + "type" : "exe", + "value" : { + "output" : [ + "[Unit]", + "Description=pgBackRest Server", + "After=network.target", + "StartLimitIntervalSec=0", + "", + "[Service]", + "Type=simple", + "Restart=always", + "RestartSec=1", + "User=postgres", + "ExecStart=/usr/bin/pgbackrest server", + "ExecReload=kill -HUP $MAINPID", + "", + "[Install]", + "WantedBy=multi-user.target" + ] + } }, { "key" : { "bash-wrap" : true, "cmd" : [ - "sudo -u pgbackrest ssh postgres@pg-standby" + "sudo systemctl enable pgbackrest" ], - "cmd-extra" : "-o StrictHostKeyChecking=no", - "host" : "repository", + "cmd-extra" : "2>&1", + "host" : "pg-standby", "load-env" : true, "output" : false, "run-as-user" : null @@ -9932,9 +10259,8 @@ "key" : { "bash-wrap" : true, "cmd" : [ - "sudo -u postgres ssh pgbackrest@repository" + "sudo systemctl start pgbackrest" ], - "cmd-extra" : "-o StrictHostKeyChecking=no", "host" : "pg-standby", "load-env" : true, "output" : false, @@ -9944,44 +10270,6 @@ }, { "key" : { - "file" : "/etc/pgbackrest/pgbackrest.conf", - "host" : "pg-standby", - "option" : { - "demo" : { - "pg1-path" : { - "value" : "/var/lib/pgsql/10/data" - } - }, - "global" : { - "log-level-file" : { - "value" : "detail" - }, - "log-level-stderr" : { - "value" : "off" - }, - "log-timestamp" : { - "value" : "n" - }, - "repo1-host" : { - "value" : "repository" - } - } - } - }, - "type" : "cfg-pgbackrest", - "value" : { - "config" : [ - "[demo]", - "pg1-path=/var/lib/pgsql/10/data", - "", - "[global]", - "log-level-file=detail", - "repo1-host=repository" - ] - } - }, - { - "key" : { "bash-wrap" : true, "cmd" : [ "sudo -u postgres mkdir -p -m 700 /var/lib/pgsql/10/data" @@ -10020,7 +10308,7 @@ "type" : "exe", "value" : { "output" : [ - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:03:36", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:47:54", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "standby_mode = 'on'" ] @@ -10140,12 +10428,13 @@ "type" : "exe", "value" : { "output" : [ - "LOG: database system was interrupted; last known up at 2021-10-29 12:03:23 UTC", + "LOG: database system was interrupted; last known up at 2021-12-31 19:47:20 UTC", "LOG: entering standby mode", "LOG: restored log file \"00000008.history\" from archive", - "LOG: restored log file \"000000080000000000000019\" from archive", - "LOG: redo starts at 0/19000028", - "LOG: consistent recovery state reached at 0/190000F8", + "LOG: restored log file \"000000080000000000000022\" from archive", + "LOG: redo starts at 0/22000028", + "LOG: restored log file \"000000080000000000000023\" from archive", + "LOG: consistent recovery state reached at 0/23000088", "LOG: database system is ready to accept read only connections" ] } @@ -10227,7 +10516,7 @@ "output" : [ " pg_switch_wal | current_timestamp ", "---------------+-------------------------------", - " 0/1A02A9C8 | 2021-10-29 12:03:40.974372+00", + " 0/2402B9C0 | 2021-12-31 19:48:05.183178+00", "(1 row)" ] } @@ -10256,7 +10545,7 @@ "output" : [ " message | current_timestamp ", "----------------+-------------------------------", - " Important Data | 2021-10-29 12:03:43.748088+00", + " Important Data | 2021-12-31 19:48:09.980843+00", "(1 row)" ] } @@ -10282,7 +10571,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: check command begin 2.36: --exec-id=1317-973dac59 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-host=repository --stanza=demo", + "P00 INFO: check command begin 2.37: --exec-id=1003-f1cf9bb3 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --stanza=demo", "P00 INFO: check repo1 (standby)", "P00 INFO: switch wal not performed because this is a standby", "P00 INFO: check command end: completed successfully" @@ -10354,10 +10643,19 @@ "[demo]", "pg1-path=/var/lib/pgsql/10/data", "recovery-option=primary_conninfo=host=172.17.0.5 port=5432 user=replicator", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", "", "[global]", "log-level-file=detail", - "repo1-host=repository" + "repo1-host=repository", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" ] } }, @@ -10429,7 +10727,7 @@ "type" : "exe", "value" : { "output" : [ - "# Recovery settings generated by pgBackRest restore on 2021-10-29 12:03:47", + "# Recovery settings generated by pgBackRest restore on 2021-12-31 19:48:19", "primary_conninfo = 'host=172.17.0.5 port=5432 user=replicator'", "restore_command = 'pgbackrest --stanza=demo archive-get %f \"%p\"'", "standby_mode = 'on'" @@ -10519,10 +10817,10 @@ "type" : "exe", "value" : { "output" : [ - " [filtered 6 lines of output]", + " [filtered 7 lines of output]", "LOG: database system is ready to accept read only connections", - "LOG: restored log file \"00000008000000000000001A\" from archive", - "LOG: started streaming WAL from primary at 0/1B000000 on timeline 8" + "LOG: restored log file \"000000080000000000000024\" from archive", + "LOG: started streaming WAL from primary at 0/25000000 on timeline 8" ] } }, @@ -10554,7 +10852,7 @@ "output" : [ " message | current_timestamp ", "----------------+-------------------------------", - " Important Data | 2021-10-29 12:03:51.037022+00", + " Important Data | 2021-12-31 19:48:26.841094+00", "(1 row)" ] } @@ -10583,7 +10881,7 @@ "output" : [ " message | current_timestamp ", "----------------+-------------------------------", - " Important Data | 2021-10-29 12:03:51.253056+00", + " Important Data | 2021-12-31 19:48:28.277963+00", "(1 row)" ] } @@ -10670,12 +10968,21 @@ "config" : [ "[demo]", "pg1-path=/var/lib/pgsql/10/data", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", "", "[global]", "archive-async=y", "log-level-file=detail", "repo1-host=repository", "spool-path=/var/spool/pgbackrest", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key", "", "[global:archive-get]", "process-max=2", @@ -10716,12 +11023,21 @@ "[demo]", "pg1-path=/var/lib/pgsql/10/data", "recovery-option=primary_conninfo=host=172.17.0.5 port=5432 user=replicator", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", "", "[global]", "archive-async=y", "log-level-file=detail", "repo1-host=repository", "spool-path=/var/spool/pgbackrest", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key", "", "[global:archive-get]", "process-max=2", @@ -10814,10 +11130,10 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: check command begin 2.36: --exec-id=5821-b35b7816 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-host=repository --stanza=demo", + "P00 INFO: check command begin 2.37: --exec-id=4666-24f1e367 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --stanza=demo", "P00 INFO: check repo1 configuration (primary)", "P00 INFO: check repo1 archive for WAL (primary)", - "P00 INFO: WAL segment 000000080000000000000020 successfully archived to '/var/lib/pgbackrest/archive/demo/10-1/0000000800000000/000000080000000000000020-1b7f6bb60c58071d54036c84af7aecacf96a699f.gz' on repo1", + "P00 INFO: WAL segment 00000008000000000000002A successfully archived to '/var/lib/pgbackrest/archive/demo/10-1/0000000800000000/00000008000000000000002A-91572e10bbcc68d082fb476a0b8492bb395f500f.gz' on repo1", "P00 INFO: check command end: completed successfully" ] } @@ -10844,24 +11160,27 @@ "value" : { "output" : [ "-------------------PROCESS START-------------------", - "P00 INFO: archive-push:async command begin 2.36: [/var/lib/pgsql/10/data/pg_wal] --archive-async --exec-id=5780-3b53e200 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: push 1 WAL file(s) to archive: 00000008000000000000001B", - "P01 DETAIL: pushed WAL file '00000008000000000000001B' to the archive", + "P00 INFO: archive-push:async command begin 2.37: [/var/lib/pgsql/10/data/pg_wal] --archive-async --exec-id=4638-a7372d44 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: push 1 WAL file(s) to archive: 000000080000000000000025", + "P01 DETAIL: pushed WAL file '000000080000000000000025' to the archive", + "P00 DETAIL: statistics: {\"socket.client\":{\"total\":1},\"socket.session\":{\"total\":1},\"tls.client\":{\"total\":1},\"tls.session\":{\"total\":1}}", "P00 INFO: archive-push:async command end: completed successfully", "", "-------------------PROCESS START-------------------", - "P00 INFO: archive-push:async command begin 2.36: [/var/lib/pgsql/10/data/pg_wal] --archive-async --exec-id=5823-440f629a --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: push 4 WAL file(s) to archive: 00000008000000000000001C...00000008000000000000001F", - "P02 DETAIL: pushed WAL file '00000008000000000000001D' to the archive", - "P01 DETAIL: pushed WAL file '00000008000000000000001C' to the archive", - "P01 DETAIL: pushed WAL file '00000008000000000000001F' to the archive", - "P02 DETAIL: pushed WAL file '00000008000000000000001E' to the archive", + "P00 INFO: archive-push:async command begin 2.37: [/var/lib/pgsql/10/data/pg_wal] --archive-async --exec-id=4668-f9468f12 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: push 4 WAL file(s) to archive: 000000080000000000000026...000000080000000000000029", + "P01 DETAIL: pushed WAL file '000000080000000000000026' to the archive", + "P02 DETAIL: pushed WAL file '000000080000000000000027' to the archive", + "P01 DETAIL: pushed WAL file '000000080000000000000028' to the archive", + "P02 DETAIL: pushed WAL file '000000080000000000000029' to the archive", + "P00 DETAIL: statistics: {\"socket.client\":{\"total\":1},\"socket.session\":{\"total\":1},\"tls.client\":{\"total\":1},\"tls.session\":{\"total\":1}}", "P00 INFO: archive-push:async command end: completed successfully", "", "-------------------PROCESS START-------------------", - "P00 INFO: archive-push:async command begin 2.36: [/var/lib/pgsql/10/data/pg_wal] --archive-async --exec-id=5835-8f656aa8 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: push 1 WAL file(s) to archive: 000000080000000000000020", - "P01 DETAIL: pushed WAL file '000000080000000000000020' to the archive", + "P00 INFO: archive-push:async command begin 2.37: [/var/lib/pgsql/10/data/pg_wal] --archive-async --exec-id=4676-d80866d0 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: push 1 WAL file(s) to archive: 00000008000000000000002A", + "P01 DETAIL: pushed WAL file '00000008000000000000002A' to the archive", + "P00 DETAIL: statistics: {\"socket.client\":{\"total\":1},\"socket.session\":{\"total\":1},\"tls.client\":{\"total\":1},\"tls.session\":{\"total\":1}}", "P00 INFO: archive-push:async command end: completed successfully" ] } @@ -10901,24 +11220,25 @@ "value" : { "output" : [ "-------------------PROCESS START-------------------", - "P00 INFO: archive-get:async command begin 2.36: [000000080000000000000019, 00000008000000000000001A, 00000008000000000000001B, 00000008000000000000001C, 00000008000000000000001D, 00000008000000000000001E, 00000008000000000000001F, 000000080000000000000020] --archive-async --exec-id=1898-07b9123d --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000019...000000080000000000000020", - "P02 DETAIL: found 00000008000000000000001A in the repo1: 10-1 archive", - "P01 DETAIL: found 000000080000000000000019 in the repo1: 10-1 archive", - "P00 DETAIL: unable to find 00000008000000000000001B in the archive", - "P00 INFO: archive-get:async command end: completed successfully", - " [filtered 14 lines of output]", - "P00 INFO: archive-get:async command begin 2.36: [00000008000000000000001B, 00000008000000000000001C, 00000008000000000000001D, 00000008000000000000001E, 00000008000000000000001F, 000000080000000000000020, 000000080000000000000021, 000000080000000000000022] --archive-async --exec-id=1953-ce2e1add --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --spool-path=/var/spool/pgbackrest --stanza=demo", - "P00 INFO: get 8 WAL file(s) from archive: 00000008000000000000001B...000000080000000000000022", - "P01 DETAIL: found 00000008000000000000001B in the repo1: 10-1 archive", - "P02 DETAIL: found 00000008000000000000001C in the repo1: 10-1 archive", - "P01 DETAIL: found 00000008000000000000001D in the repo1: 10-1 archive", - "P02 DETAIL: found 00000008000000000000001E in the repo1: 10-1 archive", - "P01 DETAIL: found 00000008000000000000001F in the repo1: 10-1 archive", - "P02 DETAIL: found 000000080000000000000020 in the repo1: 10-1 archive", - "P00 DETAIL: unable to find 000000080000000000000021 in the archive", - "P00 INFO: archive-get:async command end: completed successfully", - " [filtered 5 lines of output]" + "P00 INFO: archive-get:async command begin 2.37: [000000080000000000000022, 000000080000000000000023, 000000080000000000000024, 000000080000000000000025, 000000080000000000000026, 000000080000000000000027, 000000080000000000000028, 000000080000000000000029] --archive-async --exec-id=1485-81fe8c6f --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000022...000000080000000000000029", + "P01 DETAIL: found 000000080000000000000022 in the repo1: 10-1 archive", + "P02 DETAIL: found 000000080000000000000023 in the repo1: 10-1 archive", + "P01 DETAIL: found 000000080000000000000024 in the repo1: 10-1 archive", + "P00 DETAIL: unable to find 000000080000000000000025 in the archive", + "P00 DETAIL: statistics: {\"socket.client\":{\"total\":1},\"socket.session\":{\"total\":1},\"tls.client\":{\"total\":1},\"tls.session\":{\"total\":1}}", + " [filtered 24 lines of output]", + "P00 INFO: archive-get:async command begin 2.37: [000000080000000000000025, 000000080000000000000026, 000000080000000000000027, 000000080000000000000028, 000000080000000000000029, 00000008000000000000002A, 00000008000000000000002B, 00000008000000000000002C] --archive-async --exec-id=1505-c945be79 --log-level-console=off --log-level-file=detail --log-level-stderr=off --no-log-timestamp --pg1-path=/var/lib/pgsql/10/data --process-max=2 --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --spool-path=/var/spool/pgbackrest --stanza=demo", + "P00 INFO: get 8 WAL file(s) from archive: 000000080000000000000025...00000008000000000000002C", + "P01 DETAIL: found 000000080000000000000025 in the repo1: 10-1 archive", + "P02 DETAIL: found 000000080000000000000026 in the repo1: 10-1 archive", + "P01 DETAIL: found 000000080000000000000027 in the repo1: 10-1 archive", + "P02 DETAIL: found 000000080000000000000028 in the repo1: 10-1 archive", + "P01 DETAIL: found 000000080000000000000029 in the repo1: 10-1 archive", + "P02 DETAIL: found 00000008000000000000002A in the repo1: 10-1 archive", + "P00 DETAIL: unable to find 00000008000000000000002B in the archive", + "P00 DETAIL: statistics: {\"socket.client\":{\"total\":1},\"socket.session\":{\"total\":1},\"tls.client\":{\"total\":1},\"tls.session\":{\"total\":1}}", + " [filtered 14 lines of output]" ] } }, @@ -10949,6 +11269,18 @@ "pg2-host" : { "value" : "pg-standby" }, + "pg2-host-ca-file" : { + "value" : "/etc/pgbackrest/cert/ca.crt" + }, + "pg2-host-cert-file" : { + "value" : "/etc/pgbackrest/cert/client.crt" + }, + "pg2-host-key-file" : { + "value" : "/etc/pgbackrest/cert/client.key" + }, + "pg2-host-type" : { + "value" : "tls" + }, "pg2-path" : { "value" : "/var/lib/pgsql/10/data" } @@ -10965,8 +11297,16 @@ "config" : [ "[demo]", "pg1-host=pg-primary", + "pg1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg1-host-key-file=/etc/pgbackrest/cert/client.key", + "pg1-host-type=tls", "pg1-path=/var/lib/pgsql/10/data", "pg2-host=pg-standby", + "pg2-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg2-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg2-host-key-file=/etc/pgbackrest/cert/client.key", + "pg2-host-type=tls", "pg2-path=/var/lib/pgsql/10/data", "", "[global]", @@ -10974,7 +11314,12 @@ "process-max=3", "repo1-path=/var/lib/pgbackrest", "repo1-retention-full=2", - "start-fast=y" + "start-fast=y", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" ] } }, @@ -11001,17 +11346,21 @@ "output" : [ " [filtered 2 lines of output]", "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the requested immediate checkpoint completes", - "P00 INFO: backup start archive = 000000080000000000000022, lsn = 0/22000028", - "P00 INFO: wait for replay on the standby to reach 0/22000028", - "P00 INFO: replay on the standby reached 0/22000028", - "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/global/pg_control (8KB, 0%) checksum 636d88447b7a080b68c1a70fc65aa14f75c2665d", - "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/log/postgresql.log (5.3KB, 0%) checksum 1f669e8ef4fc8fa06c0af0fe6cb7547273e48dc5", - "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/pg_hba.conf (4.2KB, 0%) checksum 12abee43e7eabfb3ff6239f3fc9bc3598293557d", - "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/current_logfiles (26B, 0%) checksum 78a9f5c10960f0d91fcd313937469824861795a2", - "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/pg_logical/replorigin_checkpoint (8B, 0%) checksum 347fc8f2df71bd4436e38bd1516ccd7ea0d46532", - "P02 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/2608 (440KB, 19%) checksum bec9a4997a5639318eb01f2082e0b99ab3e256f6", - "P03 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/1249 (392KB, 36%) checksum 1e632c4405f74321b16b4e2edd6ac9e8b9ed03f5", - " [filtered 1254 lines of output]" + "P00 INFO: backup start archive = 00000008000000000000002C, lsn = 0/2C000028", + "P00 INFO: wait for replay on the standby to reach 0/2C000028", + "P00 INFO: replay on the standby reached 0/2C000028", + "P00 INFO: check archive for prior segment 00000008000000000000002B", + "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/global/pg_control (8KB, 0%) checksum e796914fe27be239cc661ddb5b86a893ffcd1473", + "P02 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/2608 (440KB, 19%) checksum e717affd3b3f4a27eaf400f668c98108998f5a0f", + "P03 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/1249 (392KB, 36%) checksum 42cb2ebdf4a3d5bb7a20ef360a2ce7ed9f96068c", + "P04 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/2674 (368KB, 52%) checksum 43f17b5d5a0f8fad15d1acf09b0df35cb269cd9f", + "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/log/postgresql.log (6KB, 52%) checksum daedeede97741d6a3011aaadca7f0f8db81f0d15", + "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/pg_hba.conf (4.2KB, 52%) checksum 12abee43e7eabfb3ff6239f3fc9bc3598293557d", + "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/current_logfiles (26B, 52%) checksum 78a9f5c10960f0d91fcd313937469824861795a2", + "P01 DETAIL: backup file pg-primary:/var/lib/pgsql/10/data/pg_logical/replorigin_checkpoint (8B, 52%) checksum 347fc8f2df71bd4436e38bd1516ccd7ea0d46532", + "P02 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/2673 (328KB, 66%) checksum 0835d0b60054ed006e0d61f016d235f60a8a695f", + "P03 DETAIL: backup file pg-standby:/var/lib/pgsql/10/data/base/13017/2658 (112KB, 71%) checksum d56cd71531fe675d759c3d6ad4a6b3ab20c5bc87", + " [filtered 1253 lines of output]" ] } }, @@ -11151,12 +11500,21 @@ "config" : [ "[demo]", "pg1-path=/var/lib/pgsql/11/data", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", "", "[global]", "archive-async=y", "log-level-file=detail", "repo1-host=repository", "spool-path=/var/spool/pgbackrest", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key", "", "[global:archive-get]", "process-max=2", @@ -11184,12 +11542,21 @@ "[demo]", "pg1-path=/var/lib/pgsql/11/data", "recovery-option=primary_conninfo=host=172.17.0.5 port=5432 user=replicator", + "repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "repo1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "repo1-host-key-file=/etc/pgbackrest/cert/client.key", + "repo1-host-type=tls", "", "[global]", "archive-async=y", "log-level-file=detail", "repo1-host=repository", "spool-path=/var/spool/pgbackrest", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key", "", "[global:archive-get]", "process-max=2", @@ -11224,8 +11591,16 @@ "config" : [ "[demo]", "pg1-host=pg-primary", + "pg1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg1-host-key-file=/etc/pgbackrest/cert/client.key", + "pg1-host-type=tls", "pg1-path=/var/lib/pgsql/11/data", "pg2-host=pg-standby", + "pg2-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg2-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg2-host-key-file=/etc/pgbackrest/cert/client.key", + "pg2-host-type=tls", "pg2-path=/var/lib/pgsql/11/data", "", "[global]", @@ -11233,7 +11608,12 @@ "process-max=3", "repo1-path=/var/lib/pgbackrest", "repo1-retention-full=2", - "start-fast=y" + "start-fast=y", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" ] } }, @@ -11273,7 +11653,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 INFO: stanza-upgrade command begin 2.36: --exec-id=6370-1d62bc82 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --no-online --pg1-path=/var/lib/pgsql/11/data --repo1-host=repository --stanza=demo", + "P00 INFO: stanza-upgrade command begin 2.37: --exec-id=5116-27923273 --log-level-console=info --log-level-file=detail --log-level-stderr=off --no-log-timestamp --no-online --pg1-path=/var/lib/pgsql/11/data --repo1-host=repository --repo1-host-ca-file=/etc/pgbackrest/cert/ca.crt --repo1-host-cert-file=/etc/pgbackrest/cert/client.crt --repo1-host-key-file=/etc/pgbackrest/cert/client.key --repo1-host-type=tls --stanza=demo", "P00 INFO: stanza-upgrade for stanza 'demo' on repo1", "P00 INFO: stanza-upgrade command end: completed successfully" ] @@ -11371,7 +11751,7 @@ "type" : "exe", "value" : { "output" : [ - "P00 WARN: unable to check pg-2: [DbConnectError] raised from remote-0 ssh protocol on 'pg-standby': unable to connect to 'dbname='postgres' port=5432': could not connect to server: No such file or directory", + "P00 WARN: unable to check pg-2: [DbConnectError] raised from remote-0 tls protocol on 'pg-standby': unable to connect to 'dbname='postgres' port=5432': could not connect to server: No such file or directory", " \tIs the server running locally and accepting", " \tconnections on Unix domain socket \"/var/run/postgresql/.s.PGSQL.5432\"?" ] @@ -11476,8 +11856,16 @@ "config" : [ "[demo]", "pg1-host=pg-primary", + "pg1-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg1-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg1-host-key-file=/etc/pgbackrest/cert/client.key", + "pg1-host-type=tls", "pg1-path=/var/lib/pgsql/11/data", "pg2-host=pg-standby", + "pg2-host-ca-file=/etc/pgbackrest/cert/ca.crt", + "pg2-host-cert-file=/etc/pgbackrest/cert/client.crt", + "pg2-host-key-file=/etc/pgbackrest/cert/client.key", + "pg2-host-type=tls", "pg2-path=/var/lib/pgsql/11/data", "", "[global]", @@ -11485,7 +11873,12 @@ "process-max=3", "repo1-path=/var/lib/pgbackrest", "repo1-retention-full=2", - "start-fast=y" + "start-fast=y", + "tls-server-address=*", + "tls-server-auth=pgbackrest-client=demo", + "tls-server-ca-file=/etc/pgbackrest/cert/ca.crt", + "tls-server-cert-file=/etc/pgbackrest/cert/server.crt", + "tls-server-key-file=/etc/pgbackrest/cert/server.key" ] } } diff -Nru pgbackrest-2.36/doc/resource/git-history.cache pgbackrest-2.37/doc/resource/git-history.cache --- pgbackrest-2.36/doc/resource/git-history.cache 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/resource/git-history.cache 2022-01-03 13:43:55.000000000 +0000 @@ -1,5 +1,311 @@ [ { + "commit": "62fbee72ad319f92c9410ac8dbab2f81fe945a36", + "date": "2022-01-01 10:50:16 -0500", + "subject": "Update LICENSE.txt and PostgreSQL copyright for 2022." + }, + { + "commit": "d6ebf6e2d67331a838f53beda1c186c527b56a8e", + "date": "2021-12-30 18:54:36 -0500", + "subject": "Remove dead test code." + }, + { + "commit": "fccb7f7dd45c6c373d0cfa74b90d69ca483aa3af", + "date": "2021-12-28 17:39:22 -0500", + "subject": "Add release note regarding IANA approval of the default TLS port." + }, + { + "commit": "6a12458440168f13cb05d70f36ea54b1860e390c", + "date": "2021-12-16 10:30:59 -0500", + "subject": "Parse protocol/port in S3/Azure endpoints.", + "body": "Utilize httpUrlNewParseP() to parse endpoint and port from the URL in the S3 and Azure helpers to avoid issues where protocol was not expected to be part of the URL." + }, + { + "commit": "f06101de77a980c7e4115762f2fc301280aa4127", + "date": "2021-12-16 09:47:04 -0500", + "subject": "Add TLS server documentation.", + "body": "Add documentation and make the feature visible." + }, + { + "commit": "615bdff4030a31bfedfe7df04676e3948ec9c2c0", + "date": "2021-12-14 14:53:41 -0500", + "subject": "Fix socket leak on connection retries.", + "body": "This leak was caused by the file descriptor variable getting clobbered after a long jump. Mark it as volatile to fix.\r\n\r\nTesting this is a bit complex because the issue only happens in optimized builds, if at all. Put the test into the performance suite, which is always optimized, until a better idea presents itself." + }, + { + "commit": "a73fe4eb966f9685f6e4179c397a10c1e7f15f19", + "date": "2021-12-10 15:53:40 -0500", + "subject": "Fix restore delta link mapping when path/file already exists.", + "body": "If a path/file was remapped to a link using either --link-map or --link-all there would be no affect if the path/file already existed. If a link existed it would be properly updated and converting a link to a path/file also worked.\r\n\r\nThe issue happened during delta cleanup, which failed to check if the existing path/file had been remapped to a link.\r\n\r\nAdd checks for newly mapped path/file links and remove the old path/file we required." + }, + { + "commit": "19a7ec69debfe6587fcc1163451896590c96bf21", + "date": "2021-12-10 15:04:55 -0500", + "subject": "Close expect log file when unit test completes.", + "body": "This did not cause any issues, but it is better to explicitly close open files." + }, + { + "commit": "c38e2d31709804eb4b9125a15ad84c8fc813f366", + "date": "2021-12-08 15:00:19 -0500", + "subject": "Add verb to HTTP error output.", + "body": "This makes it easier to debug HTTP errors." + }, + { + "commit": "be4ac3923cb77873da298a30aca5d847b3c635af", + "date": "2021-12-08 13:57:26 -0500", + "subject": "Error when restore is unable to find a backup to match the time target.", + "body": "This was previously a warning but the warning is easy to miss so a lot of time may be lost restoring and recovering a backup that will not hit the target.\r\n\r\nSince this is technically a breaking change, add an \"important note\" about the change to the release." + }, + { + "commit": "672330593789f07aaad90bbafcd2597cbc602686", + "date": "2021-12-08 12:29:20 -0500", + "subject": "Add warning when checkpoint_timeout exceeds db-timeout.", + "body": "In the backup command, add a warning if start-fast is disabled and the PostgreSQL checkpoint_timeout is greater than db-timeout.\r\n\r\nIn such cases, we might timeout before the checkpoint occurs and the backup really starts." + }, + { + "commit": "bd2ba802db11c505ec69943fa81b2b379073fbf4", + "date": "2021-12-08 10:16:41 -0500", + "subject": "Check that clusters are alive and correctly configured during a backup.", + "body": "Fail the backup if a cluster stops or the standby is promoted. Previously, shutting down the primary would cause an error but it was not detected until the end of the backup. Now the error will happen sooner and a promotion on the standby will also cause an error." + }, + { + "commit": "7b3ea883c7c010aafbeb14d150d073a113b703e4", + "date": "2021-12-07 18:18:43 -0500", + "subject": "Add SIGTERM and SIGHUP handling to TLS server.", + "body": "SIGHUP allows the configuration to be reloaded. Note that the configuration will not be updated in child processes that have already started.\r\n\r\nSIGTERM terminates the server process gracefully and sends SIGTERM to all child processes. This also gives the tests an easy way to stop the server." + }, + { + "commit": "49145d72bac16498cdbf5eeb3cd6128ea0be0667", + "date": "2021-12-07 09:21:07 -0500", + "subject": "Add timeline and checkpoint checks to backup.", + "body": "Add the following checks:\r\n\r\n* Checkpoint is updated in pg_control after pg_start_backup(). This helps ensure that PostgreSQL and pgBackRest have a consistent view of the storage and that PGDATA paths match.\r\n* Timeline of backup start WAL file matches pg_control. Hard to see how this one could get hit, but we have the power...\r\n* Standby is on the same timeline as the primary. If not, this standby is not following the primary.\r\n* Last standby checkpoint is not greater than the backup checkpoint. If so, this standby is not following the primary.\r\n\r\nThis also requires some additional plumbing to read/write timeline/checkpoint from pg_control and parse timelines from WAL filenames. There were some changes in the backup tests caused by the fact that pg_control now has different contents for each backup.\r\n\r\nThe check to ensure that the required checkpoint was reached on the standby should also be updated to use pg_control (it currently uses pg_control_checkpoint()), but that requires non-trivial changes to the test harness and will need to wait." + }, + { + "commit": "9c76056dd0d1d2b07a89646b087c5c8d36ab97f5", + "date": "2021-11-30 16:21:15 -0500", + "subject": "Add error type and message to CHECK() macro.", + "body": "A CHECK() worked exactly like ASSERT() except that it was compiled into production code. However, over time many checks have been added that should not throw AssertError, which should be reserved for probable coding errors.\n\nAllow the error code to be specified so other error types can be thrown. Also add a human-readable message since many of these could be seen by users even when there is no coding error.\n\nUpdate coverage exceptions for CHECK() to match ASSERT() since all conditions will never be covered." + }, + { + "commit": "0895cfcdf7d3f15b8029f73ed62c6094d30724b3", + "date": "2021-11-30 13:23:11 -0500", + "subject": "Add HRN_PG_CONTROL_PUT() and HRN_PG_CONTROL_TIME().", + "body": "These macros simplify management of pg_control test files.\n\nCentralize time updates for pg_control in the command/backup module. This caused some time updates in the logs.\n\nFinally, move the postgres module after the storage module so it can use storage macros." + }, + { + "commit": "01ac6b6cac86ea857e54a3b1c45077df1e128a75", + "date": "2021-11-30 08:28:36 -0500", + "subject": "Autogenerate test system identifiers.", + "body": "hrnPgControlToBuffer() and hrnPgWalToBuffer() now generate the system id based on the version of Postgres. If a value less than 100 is specified for systemId then it will be added to the default system id so there can be multiple ids for a single version of PostgreSQL.\n\nAdd constants to represent version system ids in tests. These will eventually be auto-generated.\n\nThis changes some checksums and we no longer have big-endian tests systems, so X those checksums out so it is obvious they are no longer valid." + }, + { + "commit": "3f7409019df112ec50efb6c3db6f7780c9a63c87", + "date": "2021-11-24 16:09:45 -0500", + "subject": "Ensure ASSERT() macro is always available in test modules.", + "body": "Tests that run without DEBUG for performance did not have ASSERT() and were using CHECK() instead.\n\nInstead ensure that the ASSERT() macro is always available in tests." + }, + { + "commit": "dcb4f09d8315e92c0877b589f3fa9b7f0fa65f93", + "date": "2021-11-23 09:37:12 -0500", + "subject": "Revert changes to backupFilePut() made in 1e77fc3d.", + "body": "These changes were made obsolete by a3d7a23a." + }, + { + "commit": "7e35245dc3416238a84a43abbecdf976170dea91", + "date": "2021-11-23 08:07:31 -0500", + "subject": "Use ASSERT() or TEST_RESULT*() instead of CHECK() in test modules." + }, + { + "commit": "a3d7a23a9d90611a3d31947598fbea240b250710", + "date": "2021-11-22 12:52:37 -0500", + "subject": "Use infoBackupDataByLabel() to log backup size.", + "body": "Eliminate summing and passing of copied files sizes for logging backup size.\r\n\r\nInstead, utilize infoBackupDataByLabel() to pull the backup size for the log message." + }, + { + "commit": "1a0560d363d28737befb8c222647783d4fc2ca29", + "date": "2021-11-19 12:22:09 -0500", + "subject": "Allow y/n arguments for boolean command-line options.", + "body": "This allows boolean boolean command-line options to work like their config file equivalents.\r\n\r\nAt least for now this behavior will remain undocumented since all examples in the documentation will continue to use the standard syntax. The idea is that it will \"just work\" when options are copied out of config files rather than generating an error." + }, + { + "commit": "2d963ce9471808172f879916c3f3accc35f14d56", + "date": "2021-11-18 17:23:11 -0500", + "subject": "Rename server-start command to server." + }, + { + "commit": "1f14f45dfb0d1677a695719381cbd5a8a3c6c986", + "date": "2021-11-18 16:18:10 -0500", + "subject": "Check archive immediately after backup start.", + "body": "Previously the archive was only checked at the end of the backup to ensure all WAL required to make the backup consistent was present. The problem was that if archiving was not functioning then the backup had to complete before the user found out, which could be a while if the database was large enough.\r\n\r\nAdd an archive check immediately after backup start so failures are reported earlier.\r\n\r\nThe trick is to determine which WAL to check. If the repo is new there may not be any WAL in it and pg_start_backup() will not switch the WAL segment if it is empty. These are both likely scenarios when setting up and/or testing pgBackRest.\r\n\r\nIf the WAL segment is switched by pg_start_backup(), then check the archive for the segment that was detected prior to backup start. This should be common on normal running clusters with regular activity. Note that this might not be the segment immediately prior to the backup start segment if WAL volume is high.\r\n\r\nIf pg_start_backup() did not switch the WAL then we can force a switch on PostgreSQL >= 9.3 by creating a restore point. In that case the WAL to check will be the backup start WAL. This is most likely to happen on idle systems, during testing, or immediately after a repo switch.\r\n\r\nAn advantage of this approach other than earlier notification is that the backup directory will not be created so no resume will be attempted on the next backup.\r\n\r\nNote that some additional churn was created in backup.c because the load of archive.info needs to be done earlier." + }, + { + "commit": "dea752477ab8e812cdbd717eb2091baf3f5d0906", + "date": "2021-11-17 16:39:04 -0500", + "subject": "Remove obsolete statement about future multi-repository support." + }, + { + "commit": "0949b4d35fdd04c55927eb6a107d881376dbe73c", + "date": "2021-11-16 18:26:21 -0500", + "subject": "Add linefeed and remove space." + }, + { + "commit": "809f0bbc638cdd95540e2257383147919f82e8f9", + "date": "2021-11-16 11:34:53 -0500", + "subject": "Add infoBackupLabelExists().", + "body": "This is easier to read than using infoBackupDataByLabel() != NULL.\n\nIt also allows an assertion to be added to infoBackupDataByLabel() to ensure that a NULL return value is not used unsafely." + }, + { + "commit": "1e77fc3d75490b7a1b6a0b31be9298c995ec672f", + "date": "2021-11-16 10:21:32 -0500", + "subject": "Include backup_label and tablespace_map file sizes in log output.", + "body": "In cases where they are returned by postgres, include backup_label and tablespace_map file sizes in the backup size value output in the log." + }, + { + "commit": "6b5322cdad7163d91b43d37d9d8eeaa39ac7f214", + "date": "2021-11-16 09:27:15 -0500", + "subject": "Add findutils package to RHEL 8 documentation container.", + "body": "This package was dropped from the most recent Rocky Linux 8 image." + }, + { + "commit": "df89eff429e9b8fbc68d9e9895badf9719fd31d2", + "date": "2021-11-15 16:53:41 -0500", + "subject": "Fix typos and improve documentation for the tablespace-map-all option." + }, + { + "commit": "fcae9d35038d454c674921c65beb02b195981480", + "date": "2021-11-15 16:42:46 -0500", + "subject": "Fix parameter test logging in parseOptionIdxValue()." + }, + { + "commit": "b3a5f7a8e27768c445458e47dad626609814fbb7", + "date": "2021-11-15 14:32:22 -0500", + "subject": "Add tablespace_map file to command/backup test module.", + "body": "The code worked fine but better to have explicit tests for this file." + }, + { + "commit": "e62ba8e85eaf469052960c4fd71ffaf26c1a1baa", + "date": "2021-11-12 17:15:45 -0500", + "subject": "Add path to pgbench used for stress test in user guide.", + "body": "This allows the stress test to run on RHEL." + }, + { + "commit": "43cfa9cef776360e592882c0b787704dbeb36cb3", + "date": "2021-11-10 12:14:41 -0500", + "subject": "Revive archive performance test.", + "body": "This test was lost due to a syntax issue in a58635ac.\n\nUpdate the test to use system() to better mimic what postgres does and add logging so pgBackRest timing can be determined." + }, + { + "commit": "dd96c29f963609fad38dac3349d7fa41e40722bb", + "date": "2021-11-10 07:53:46 -0500", + "subject": "Refactor postgres/client module with inline getters/setters.", + "body": "Extend the pattern introduced in 79a2d02 to the postgres/client module." + }, + { + "commit": "afe77e76e0adf948138d797e227a6f4c7d47c2eb", + "date": "2021-11-10 07:31:02 -0500", + "subject": "Update contributor for 6e635764." + }, + { + "commit": "6e635764a66278d5a8c2b4d30b23063bc3923067", + "date": "2021-11-09 13:24:56 -0500", + "subject": "Match backup log size with size reported by info command.", + "body": "Properly log the size of files copied during the backup, matching the backup size returned from the info command.\r\n\r\nIn the reference issue, the incremental backup after switchover logs the size of all files evaluated rather than only the size of the files copied in the backup." + }, + { + "commit": "d05d6b87142347cb4891304833db389dcf7f9a81", + "date": "2021-11-08 09:39:58 -0500", + "subject": "Do not delete manifests individually during stanza delete.", + "body": "This appears to have been an attempt to not delete files that we don't recognize, but it only works in narrow cases and could leave the user is a position of not being able to complete the stanza delete without manual intervention. It seems better just to proceed with the delete, especially since the info files have already been removed.\n\nIn addition, deleting the manifests individually could be slow on object stores if there were a very large number of backups." + }, + { + "commit": "bb03b3f41942d0b781931092a76877ad309001ef", + "date": "2021-11-04 09:44:31 -0400", + "subject": "Refactor switch statements in strIdBitFromZN().", + "body": "Coverity does not like fall-throughs either to or from the default case so refactor to avoid that." + }, + { + "commit": "676b9d95dd2467d4bddd402b5cd2b4f445c71944", + "date": "2021-11-04 08:19:18 -0400", + "subject": "Optional parameters for tlsClientNew().", + "body": "There are a number of optional parameters with the same type so this makes them easier to track and reduces churn when new ones are added." + }, + { + "commit": "038abaa71d816cc87b382bd81d3df62ddec9455a", + "date": "2021-11-03 15:23:08 -0400", + "subject": "Display size option default and allowed values with appropriate units.", + "body": "Size option default and allowed values were displayed in bytes, which was confusing for the user.\r\n\r\nThis also lays the groundwork for adding units to time options.\r\n\r\nMove option parsing functions into a common module so they can be used from the build module." + }, + { + "commit": "1b93a772369bbb3a936099e0d9d5cc79bad1e0f6", + "date": "2021-11-03 12:14:17 -0400", + "subject": "Use void * instead of List * to avoid Coverity false positives.", + "body": "Coverity complains that this should be \"List\" but that is clearly not correct." + }, + { + "commit": "2a576477b316238473525e56bc8fc8ea5790455f", + "date": "2021-11-03 11:36:34 -0400", + "subject": "Add --cmd option.", + "body": "Allows users to provide an executable to be used when pgbackrest generates command strings that expect to invoke pgbackrest. These generated commands are written to files by pgbackrest, e.g. recovery.conf." + }, + { + "commit": "c5b5b5880619d0994ab4a8feb3f60ab52170b61b", + "date": "2021-11-03 10:36:31 -0400", + "subject": "Simplify error handler.", + "body": "The error handler used a loop to process try, catch, and finally blocks. This worked fine but static analysis tools like Coverity did not understand that the finally block would always run and so there were false positives about double-free, unfreed resource, etc.\r\n\r\nThis implementation removes the loop, which simplifies everything, and makes it clear that the finally block will always run. This cuts down on Coverity false positives.\r\n\r\nThis implementation also catches lack of coverage on empty catch blocks so a few test fixes were committed separately in d74fe7a.\r\n\r\nA small refactor in backup.c is required because gcc 10.3.1 on Fedora 33 complains that the reason variable may be used uninitialized. It's not clear why this is the case, but reducing the scope of the TRY block fixes the issue." + }, + { + "commit": "cff961ede7e41fa8035ffe7451a22eb5ea0e46c1", + "date": "2021-11-03 07:38:06 -0400", + "subject": "Centralize logic to build value lists during config rendering.", + "body": "This reduces duplication and makes it easier to add new types." + }, + { + "commit": "7f6c513be925c77bc6a177408efcf79f624ffc94", + "date": "2021-11-03 07:27:26 -0400", + "subject": "Add StringId as an option type.", + "body": "Rather the converting String to StringIds at runtime, store defaults in StringId format in parse.auto.c and convert user input to StringId during parsing." + }, + { + "commit": "b13844086d419dc3070bcce4e918b2353bf4887c", + "date": "2021-11-01 17:35:19 -0400", + "subject": "Use cfgOptionStrId() instead of cfgOptionStr() where appropriate.", + "body": "The compress-type, repo-type and log-level-* options have allow lists, which means it is more efficient to treat them as StringIds.\r\n\r\nFor compress-type and log-level-* also update the functions that convert them to enums." + }, + { + "commit": "b237d0cd592bbc6c6ee9280fb7aed264bf79eb9d", + "date": "2021-11-01 10:43:08 -0400", + "subject": "Remove placeholder bz2 helper data.", + "body": "This placeholder data should have been removed when bz2 support was added in a021c9fe053." + }, + { + "commit": "f4e281399a81835821547ea5c78ed7a189914d3d", + "date": "2021-11-01 10:27:57 -0400", + "subject": "Remove unused protocol log level.", + "body": "This log level was used in the Perl code but was never ported to C." + }, + { + "commit": "bc352fa6a8cff7cc08b6c7f3cdfac664d2b0805f", + "date": "2021-11-01 10:08:56 -0400", + "subject": "Simplify strIdFrom*() functions.", + "body": "The strIdFrom*() forced the caller to pick an encoding, which led to a number of TRY...CATCH blocks in the code. In practice the caller does not care which encoding is used as long as the string is valid for some encoding.\r\n\r\nUpdate the strIdFrom*() function to try all possible encodings and only throw an error when the string is not valid for any of them." + }, + { + "commit": "a92d7938197d1035e362390ce467ae827cbae051", + "date": "2021-11-01 09:11:43 -0400", + "subject": "Update automake version.", + "body": "There were no changes to install.sh in this version." + }, + { + "commit": "904b897f5e89542784af64b364a49205e7a6e040", + "date": "2021-11-01 09:03:42 -0400", + "subject": "Begin v2.37 development." + }, + { + "commit": "42fd6ce4e09ee92614cfbfb6766d9c3a6ba9cc1a", + "date": "2021-11-01 08:59:14 -0400", + "subject": "v2.36: Minor Bug Fixes and Improvements" + }, + { "commit": "6abb06248c2829f2c27a7a553d373b0fdf70cfc3", "date": "2021-10-29 11:45:50 -0400", "subject": "Make analytics optional for HTML documentation.", diff -Nru pgbackrest-2.36/doc/xml/auto/metric-coverage-report.auto.xml pgbackrest-2.37/doc/xml/auto/metric-coverage-report.auto.xml --- pgbackrest-2.36/doc/xml/auto/metric-coverage-report.auto.xml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/xml/auto/metric-coverage-report.auto.xml 2022-01-03 13:43:55.000000000 +0000 @@ -2,14 +2,14 @@ build/common 8/8 (100.0%) 10/10 (100.0%) - 101/101 (100.0%) + 97/97 (100.0%) build/config - 29/29 (100.0%) - 454/454 (100.0%) - 914/914 (100.0%) + 30/30 (100.0%) + 432/432 (100.0%) + 903/903 (100.0%) @@ -44,21 +44,21 @@ command/archive/get 9/9 (100.0%) 192/192 (100.0%) - 464/464 (100.0%) + 468/468 (100.0%) command/archive/push 12/12 (100.0%) 130/130 (100.0%) - 403/403 (100.0%) + 407/407 (100.0%) command/backup - 34/34 (100.0%) - 468/468 (100.0%) - 1142/1142 (100.0%) + 35/35 (100.0%) + 484/484 (100.0%) + 1172/1172 (100.0%) @@ -79,13 +79,13 @@ command/expire 10/10 (100.0%) 240/240 (100.0%) - 395/395 (100.0%) + 396/396 (100.0%) command/help 5/5 (100.0%) - 144/144 (100.0%) + 132/132 (100.0%) 241/241 (100.0%) @@ -93,7 +93,7 @@ command/info 12/12 (100.0%) 304/304 (100.0%) - 650/650 (100.0%) + 654/654 (100.0%) @@ -107,49 +107,49 @@ command/remote 1/1 (100.0%) 6/6 (100.0%) - 19/19 (100.0%) + 20/20 (100.0%) command/repo 8/8 (100.0%) 108/108 (100.0%) - 211/211 (100.0%) + 213/213 (100.0%) command/restore 27/27 (100.0%) - 504/504 (100.0%) - 1038/1038 (100.0%) + 506/506 (100.0%) + 1034/1034 (100.0%) command/server - 2/2 (100.0%) - 10/10 (100.0%) - 39/39 (100.0%) + 6/6 (100.0%) + 24/24 (100.0%) + 87/87 (100.0%) command/stanza - 7/7 (100.0%) - 108/108 (100.0%) - 162/162 (100.0%) + 6/6 (100.0%) + 106/106 (100.0%) + 149/149 (100.0%) command/verify 20/20 (100.0%) 268/268 (100.0%) - 700/700 (100.0%) + 702/702 (100.0%) common - 171/171 (100.0%) - 558/558 (100.0%) - 1873/1873 (100.0%) + 169/169 (100.0%) + 548/548 (100.0%) + 1866/1866 (100.0%) @@ -212,56 +212,56 @@ common/io/http 45/45 (100.0%) 240/240 (100.0%) - 694/694 (100.0%) + 695/695 (100.0%) common/io/socket 22/22 (100.0%) - 32/32 (100.0%) - 284/284 (100.0%) + 34/34 (100.0%) + 286/286 (100.0%) common/io/tls 34/34 (100.0%) 98/98 (100.0%) - 512/512 (100.0%) + 514/514 (100.0%) common/type - 281/281 (100.0%) - 796/796 (100.0%) - 3628/3628 (100.0%) + 282/282 (100.0%) + 802/802 (100.0%) + 3640/3640 (100.0%) config 88/88 (100.0%) - 860/860 (100.0%) - 1709/1709 (100.0%) + 874/874 (100.0%) + 1716/1716 (100.0%) db - 23/23 (100.0%) - 98/98 (100.0%) - 374/374 (100.0%) + 24/24 (100.0%) + 128/128 (100.0%) + 429/429 (100.0%) info 88/88 (100.0%) 770/770 (100.0%) - 2246/2246 (100.0%) + 2249/2249 (100.0%) postgres - 29/29 (100.0%) + 30/30 (100.0%) 96/96 (100.0%) - 357/357 (100.0%) + 367/367 (100.0%) @@ -275,7 +275,7 @@ protocol 55/55 (100.0%) 232/232 (100.0%) - 932/932 (100.0%) + 938/938 (100.0%) @@ -288,8 +288,8 @@ storage/azure 24/24 (100.0%) - 104/104 (100.0%) - 499/499 (100.0%) + 106/106 (100.0%) + 505/505 (100.0%) @@ -323,13 +323,13 @@ storage/s3 29/29 (100.0%) - 132/132 (100.0%) - 670/670 (100.0%) + 134/134 (100.0%) + 677/677 (100.0%) TOTAL - 1436/1436 (100.0%) - 8343/8344 (99.99%) - 25945/25945 (100.0%) + 1442/1442 (100.0%) + 8385/8386 (99.99%) + 26113/26113 (100.0%) \ No newline at end of file diff -Nru pgbackrest-2.36/doc/xml/index.xml pgbackrest-2.37/doc/xml/index.xml --- pgbackrest-2.36/doc/xml/index.xml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/xml/index.xml 2022-01-03 13:43:55.000000000 +0000 @@ -50,7 +50,7 @@
Local or Remote Operation -

A custom protocol allows to backup, restore, and archive locally or remotely via SSH with minimal configuration. An interface to query is also provided via the protocol layer so that remote access to is never required, which enhances security.

+

A custom protocol allows to backup, restore, and archive locally or remotely via TLS/SSH with minimal configuration. An interface to query is also provided via the protocol layer so that remote access to is never required, which enhances security.

diff -Nru pgbackrest-2.36/doc/xml/release.xml pgbackrest-2.37/doc/xml/release.xml --- pgbackrest-2.36/doc/xml/release.xml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/xml/release.xml 2022-01-03 13:43:55.000000000 +0000 @@ -14,6 +14,269 @@ + + + +

IMPORTANT NOTE: If the restore command is unable to find a backup that matches a specified time target then an error will be thrown, whereas before a warning was logged.

+ +

IMPORTANT NOTE: The default TLS server port has not yet been approved by IANA and is subject to change once the approval process is complete.

+ + + + + + + + + + + + + +

Fix restore delta link mapping when path/file already exists.

+ + + + + + + + + + + + +

Fix socket leak on connection retries.

+ + + + + + + + + + + + + + + + + + + + + +

Add TLS server.

+ + + + + + + + + + + + + +

Add --cmd option.

+ + + + + + + + + + + + + + +

Check archive immediately after backup start.

+ + + + + + + + + + + +

Add timeline and checkpoint checks to backup.

+ + + + + + + + + + +

Check that clusters are alive and correctly configured during a backup.

+ + + + + + + + + + + + + +

Error when restore is unable to find a backup to match the time target.

+ + + + + + + + + + + +

Parse protocol/port in S3/Azure endpoints.

+ + + + + + + + + + +

Add warning when checkpoint_timeout exceeds db-timeout.

+ + + + + + + + + + +

Add verb to HTTP error output.

+ + + + + + + + + + +

Allow y/n arguments for boolean command-line options.

+ + + + + + + + + + + + + + + + + + + + + + +

Make backup size logging exactly match info command output.

+ + + + + + + + + + + + +

Simplify error handler.

+ + + + + + + + + + + + + + + + + + +

Add StringId as an option type.

+ + + + + + + + + + + + + + + + + +

Display size option default and allowed values with appropriate units.

+ + + + + + + + + +

Fix typos and improve documentation for the tablespace-map-all option.

+ + + + + + + + +

Remove obsolete statement about future multi-repository support.

+ + + + + @@ -176,7 +439,7 @@ -

Add TLS server.

+

Add TLS server.

@@ -10252,7 +10515,7 @@ Christoph Berg - ChristophBerg + df7cb @@ -10631,6 +10894,11 @@ mhagander + + Mahomed Hussein + mahomedh + + Oscar MannerMan @@ -10889,6 +11157,11 @@ vitabaks + + Virgile CREVON + collectionneurfou + + vthriller vthriller diff -Nru pgbackrest-2.36/doc/xml/user-guide.xml pgbackrest-2.37/doc/xml/user-guide.xml --- pgbackrest-2.36/doc/xml/user-guide.xml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/doc/xml/user-guide.xml 2022-01-03 13:43:55.000000000 +0000 @@ -60,6 +60,10 @@ resource/fake-cert {[host-repo-path]}/doc/{[fake-cert-path-relative]} + + '{[os-type]}' eq '{[os-debian]}' + '{[os-type]}' eq '{[os-rhel]}' + /build {[build-path]}/pgbackrest-release-{[version]} @@ -346,7 +350,7 @@ VOLUME [ "/sys/fs/cgroup" ] # Install packages - RUN yum install -y openssh-server openssh-clients sudo wget vim dnf-plugins-core 2>&1 + RUN yum install -y openssh-server openssh-clients sudo wget vim openssl findutils dnf-plugins-core 2>&1 # Enable PowerTools repository (only available on RHEL8) RUN dnf config-manager --set-enabled powertools || true @@ -396,7 +400,72 @@ -

requires passwordless SSH to enable communication between the hosts.

+

can use passwordless SSH to enable communication between the hosts. It is also possible to use TLS, see Setup TLS.

+ + + + + Setup pgBackRest Server + + + + mkdir -p -m 770 /etc/pgbackrest/cert && + cp {[pgbackrest-repo-path]}/doc/{[fake-cert-path-relative]}/ca.crt + /etc/pgbackrest/cert/ca.crt && + + openssl genrsa -out /etc/pgbackrest/cert/server.key 2048 2>&1 && + chmod 600 /etc/pgbackrest/cert/server.key && + openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/server.csr + -key /etc/pgbackrest/cert/server.key -subj "/CN={[setup-tls-host]}" 2>&1 && + openssl x509 -req -in /etc/pgbackrest/cert/server.csr + -CA /etc/pgbackrest/cert/ca.crt + -CAkey {[pgbackrest-repo-path]}/doc/{[fake-cert-path-relative]}/ca.key -CAcreateserial + -out /etc/pgbackrest/cert/server.crt -days 9 2>&1 && + + openssl genrsa -out /etc/pgbackrest/cert/client.key 2048 2>&1 && + chmod 600 /etc/pgbackrest/cert/client.key && + openssl req -new -sha256 -nodes -out /etc/pgbackrest/cert/client.csr + -key /etc/pgbackrest/cert/client.key -subj "/CN=pgbackrest-client" 2>&1 && + openssl x509 -req -in /etc/pgbackrest/cert/client.csr + -CA /etc/pgbackrest/cert/ca.crt + -CAkey {[pgbackrest-repo-path]}/doc/{[fake-cert-path-relative]}/ca.key -CAcreateserial + -out /etc/pgbackrest/cert/client.crt -days 9 2>&1 && + + chown -R {[setup-tls-user]} /etc/pgbackrest/cert + + + + + + echo '[Unit]' | tee /etc/systemd/system/pgbackrest.service && + echo 'Description=pgBackRest Server' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'After=network.target' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'StartLimitIntervalSec=0' | tee -a /etc/systemd/system/pgbackrest.service && + echo '' | tee -a /etc/systemd/system/pgbackrest.service && + echo '[Service]' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'Type=simple' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'Restart=always' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'RestartSec=1' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'User={[setup-tls-user]}' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'ExecStart=/usr/bin/pgbackrest server' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'ExecReload=kill -HUP $MAINPID' | tee -a /etc/systemd/system/pgbackrest.service && + echo '' | tee -a /etc/systemd/system/pgbackrest.service && + echo '[Install]' | tee -a /etc/systemd/system/pgbackrest.service && + echo 'WantedBy=multi-user.target' | tee -a /etc/systemd/system/pgbackrest.service + + + + cat /etc/systemd/system/pgbackrest.service + + + + systemctl enable pgbackrest + 2>&1 + + + systemctl start pgbackrest + + @@ -780,7 +849,7 @@

Many option names have changed to improve consistency although the old names from v1 are still accepted. In general, db-* options have been renamed to pg-* and backup-*/retention-* options have been renamed to repo-* when appropriate.

-

and repository options must be indexed when using the new names introduced in v2, e.g. pg1-host, pg1-path, repo1-path, repo1-type, etc. Only one repository is allowed currently but more flexibility is planned for v2.

+

and repository options must be indexed when using the new names introduced in v2, e.g. pg1-host, pg1-path, repo1-path, repo1-type, etc.

@@ -2304,7 +2373,7 @@ -
+
Setup Passwordless SSH @@ -2343,6 +2412,10 @@
Configuration +

can use TLS with client certificates to enable communication between the hosts. It is also possible to use SSH, see Setup SSH.

+ +

expects client/server certificates to be generated in the same way as . See Secure TCP/IP Connections with TLS for detailed instructions on generating certificates.

+ Configure the <backrest/> repository path @@ -2358,6 +2431,17 @@ {[host-pg1]} + tls + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/client.crt + /etc/pgbackrest/cert/client.key + + pgbackrest-client=demo + * + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/server.crt + /etc/pgbackrest/cert/server.key + y 2 @@ -2374,6 +2458,17 @@ {[host-repo1]} + tls + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/client.crt + /etc/pgbackrest/cert/client.key + + pgbackrest-client=demo + * + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/server.crt + /etc/pgbackrest/cert/server.key + detail off @@ -2426,6 +2521,28 @@ {[br-user]}:{[br-group]} n +
+ +
+ Setup TLS Server + +

The TLS server must be configured and started on each host.

+ + + {[host-repo1]} + {[br-user]} + {[br-group]} + + + + {[host-pg1]} + postgres + postgres + +
+ +
+ Create and Check Stanza

Create the stanza in the new repository.

@@ -2656,7 +2773,7 @@
-
+
Setup Passwordless SSH @@ -2692,12 +2809,29 @@ {[host-repo1]} + tls + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/client.crt + /etc/pgbackrest/cert/client.key + + pgbackrest-client=demo + * + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/server.crt + /etc/pgbackrest/cert/server.key + detail off n + + {[host-pg2]} + postgres + postgres + +

The demo cluster must be created (even though it will be overwritten on restore) in order to create the configuration files.

@@ -3162,7 +3296,12 @@ Configure <br-option>pg2-host</br-option>/<br-option>pg2-host-user</br-option> and <br-option>pg2-path</br-option> {[pg-path]} + tls {[host-pg2]} + /etc/pgbackrest/cert/ca.crt + /etc/pgbackrest/cert/client.crt + /etc/pgbackrest/cert/client.key + @@ -3323,7 +3462,7 @@ Load data - pgbench -n -i -s {[stress-scale-data]} + {[pg-bin-path]}/pgbench -n -i -s {[stress-scale-data]} 2>&1 diff -Nru pgbackrest-2.36/LICENSE pgbackrest-2.37/LICENSE --- pgbackrest-2.36/LICENSE 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/LICENSE 2022-01-03 13:43:55.000000000 +0000 @@ -1,7 +1,7 @@ The MIT License (MIT) -Portions Copyright (c) 2015-2021, The PostgreSQL Global Development Group -Portions Copyright (c) 2013-2021, David Steele +Portions Copyright (c) 2015-2022, The PostgreSQL Global Development Group +Portions Copyright (c) 2013-2022, David Steele Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff -Nru pgbackrest-2.36/README.md pgbackrest-2.37/README.md --- pgbackrest-2.36/README.md 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/README.md 2022-01-03 13:43:55.000000000 +0000 @@ -4,7 +4,7 @@ pgBackRest aims to be a reliable, easy-to-use backup and restore solution that can seamlessly scale up to the largest databases and workloads by utilizing algorithms that are optimized for database-specific requirements. -pgBackRest [v2.36](https://github.com/pgbackrest/pgbackrest/releases/tag/release/2.36) is the current stable release. Release notes are on the [Releases](http://www.pgbackrest.org/release.html) page. +pgBackRest [v2.37](https://github.com/pgbackrest/pgbackrest/releases/tag/release/2.37) is the current stable release. Release notes are on the [Releases](http://www.pgbackrest.org/release.html) page. Please find us on [GitHub](https://github.com/pgbackrest/pgbackrest) and give us a star if you like pgBackRest! @@ -18,7 +18,7 @@ ### Local or Remote Operation -A custom protocol allows pgBackRest to backup, restore, and archive locally or remotely via SSH with minimal configuration. An interface to query PostgreSQL is also provided via the protocol layer so that remote access to PostgreSQL is never required, which enhances security. +A custom protocol allows pgBackRest to backup, restore, and archive locally or remotely via TLS/SSH with minimal configuration. An interface to query PostgreSQL is also provided via the protocol layer so that remote access to PostgreSQL is never required, which enhances security. ### Multiple Repositories diff -Nru pgbackrest-2.36/src/build/common/render.c pgbackrest-2.37/src/build/common/render.c --- pgbackrest-2.36/src/build/common/render.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/common/render.c 2022-01-03 13:43:55.000000000 +0000 @@ -9,17 +9,7 @@ String * bldStrId(const char *const buffer) { - StringId result = 0; - - TRY_BEGIN() - { - result = strIdFromZ(stringIdBit5, buffer); - } - CATCH_ANY() - { - result = strIdFromZ(stringIdBit6, buffer); - } - TRY_END(); + StringId result = strIdFromZ(buffer); return strNewFmt("STRID%u(\"%s\", 0x%" PRIx64 ")", (unsigned int)(result & STRING_ID_BIT_MASK) + 5, buffer, result); } diff -Nru pgbackrest-2.36/src/build/common/yaml.c pgbackrest-2.37/src/build/common/yaml.c --- pgbackrest-2.36/src/build/common/yaml.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/common/yaml.c 2022-01-03 13:43:55.000000000 +0000 @@ -54,15 +54,15 @@ *this = (Yaml){{0}}; // Extra braces are required for older gcc versions // Initialize parser context - CHECK(yaml_parser_initialize(&this->parser)); + CHECK(ServiceError, yaml_parser_initialize(&this->parser), "unable to initialize yaml parser"); memContextCallbackSet(objMemContext(this), yamlFreeResource, this); // Set yaml string yaml_parser_set_input_string(&this->parser, bufPtrConst(buffer), bufUsed(buffer)); // Start document - CHECK(yamlEventNext(this).type == yamlEventTypeStreamBegin); - CHECK(yamlEventNext(this).type == yamlEventTypeDocBegin); + CHECK(FormatError, yamlEventNext(this).type == yamlEventTypeStreamBegin, "expected yaml stream begin"); + CHECK(FormatError, yamlEventNext(this).type == yamlEventTypeDocBegin, "expected yaml document begin"); } OBJ_NEW_END(); @@ -111,7 +111,7 @@ FUNCTION_TEST_RETURN(yamlEventTypeMapEnd); default: - CHECK(type == YAML_NO_EVENT); + CHECK(FormatError, type == YAML_NO_EVENT, "expected yaml no event"); FUNCTION_TEST_RETURN(yamlEventTypeNone); } } @@ -130,7 +130,7 @@ if (!yaml_parser_parse(&this->parser, &event)) { // These should always be set - CHECK(this->parser.problem_mark.line && this->parser.problem_mark.column); + CHECK(ServiceError, this->parser.problem_mark.line && this->parser.problem_mark.column, "invalid yaml error info"); THROW_FMT( FormatError, "yaml parse error: %s at line: %lu column: %lu", this->parser.problem, diff -Nru pgbackrest-2.36/src/build/config/config.yaml pgbackrest-2.37/src/build/config/config.yaml --- pgbackrest-2.36/src/build/config/config.yaml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/config/config.yaml 2022-01-03 13:43:55.000000000 +0000 @@ -119,14 +119,12 @@ local: {} remote: {} + server: {} + server-ping: - internal: true log-file: false parameter-allowed: true - server-start: - internal: true - stanza-create: command-role: remote: {} @@ -300,7 +298,7 @@ main: {} output: - type: string + type: string-id default: text command: info: {} @@ -372,7 +370,7 @@ main: {} target-action: - type: string + type: string-id default: pause command: restore: {} @@ -423,6 +421,7 @@ main: {} type: + type: string-id command: backup: allow-list: @@ -444,7 +443,6 @@ default: default command-role: main: {} - type: string # Command-line only local/remote options #--------------------------------------------------------------------------------------------------------------------------------- @@ -479,7 +477,7 @@ remote: {} remote-type: - type: string + type: string-id internal: true required: false allow-list: @@ -552,7 +550,7 @@ main: {} sort: - type: string + type: string-id default: asc allow-list: - none @@ -581,19 +579,19 @@ buffer-size: section: global type: size - default: 1048576 + default: 1MiB allow-list: - - 16384 - - 32768 - - 65536 - - 131072 - - 262144 - - 524288 - - 1048576 - - 2097152 - - 4194304 - - 8388608 - - 16777216 + - 16KiB + - 32KiB + - 64KiB + - 128KiB + - 256KiB + - 512KiB + - 1MiB + - 2MiB + - 4MiB + - 8MiB + - 16MiB command: archive-get: {} archive-push: {} @@ -607,17 +605,17 @@ repo-put: {} repo-rm: {} restore: {} + server: {} server-ping: {} - server-start: {} stanza-create: {} stanza-delete: {} stanza-upgrade: {} verify: {} - cmd-ssh: + cmd: section: global type: string - default: ssh + required: false command: archive-get: {} archive-push: {} @@ -639,6 +637,11 @@ main: {} local: {} + cmd-ssh: + inherit: cmd + required: true + default: ssh + # Option is deprecated and should not be referenced outside of cfgLoadUpdateOption() compress: section: global @@ -686,7 +689,7 @@ compress-type: section: global - type: string + type: string-id default: gz allow-list: - none @@ -832,7 +835,7 @@ repo-put: {} repo-rm: {} restore: {} - server-start: {} + server: {} stanza-create: {} stanza-delete: {} stanza-upgrade: {} @@ -901,65 +904,51 @@ depend: tcp-keep-alive-count tls-server-ca-file: - internal: true section: global type: path command: - server-start: - internal: false + server: {} tls-server-cert-file: - internal: true section: global type: path command: - server-start: - internal: false + server: {} tls-server-key-file: - internal: true section: global type: path command: - server-start: - internal: false + server: {} tls-server-auth: - internal: true section: global type: hash command: - server-start: - internal: false + server: {} tls-server-address: - internal: true section: global type: string default: localhost command: - server-ping: - internal: false - server-start: - internal: false + server: {} + server-ping: {} tls-server-port: - internal: true section: global type: integer default: 8432 allow-range: [1, 65535] command: - server-ping: - internal: false - server-start: - internal: false + server: {} + server-ping: {} # Logging options #--------------------------------------------------------------------------------------------------------------------------------- log-level-console: section: global - type: string + type: string-id default: warn allow-list: - off @@ -972,13 +961,13 @@ log-level-file: section: global - type: string + type: string-id default: info allow-list: log-level-console log-level-stderr: section: global - type: string + type: string-id default: warn allow-list: log-level-console @@ -1027,8 +1016,8 @@ archive-get-queue-max: section: global type: size - default: 134217728 - allow-range: [0, 4503599627370496] + default: 128MiB + allow-range: [0, 4PiB] command: archive-get: {} command-role: @@ -1049,7 +1038,7 @@ section: global type: size required: false - allow-range: [0, 4503599627370496] + allow-range: [0, 4PiB] command: archive-push: {} command-role: @@ -1143,8 +1132,8 @@ manifest-save-threshold: section: global type: size - default: 1073741824 - allow-range: [1, 1099511627776] + default: 1GiB + allow-range: [1, 1TiB] command: backup: {} command-role: @@ -1181,7 +1170,7 @@ #--------------------------------------------------------------------------------------------------------------------------------- archive-mode: section: global - type: string + type: string-id default: preserve allow-list: - off @@ -1332,7 +1321,6 @@ db?-host: {} pg-host-cert-file: - internal: true section: global group: pg type: string @@ -1417,10 +1405,9 @@ db?-user: {} pg-host-type: - internal: true section: global group: pg - type: string + type: string-id default: ssh allow-list: - ssh @@ -1602,7 +1589,7 @@ repo-type: section: global group: repo - type: string + type: string-id default: posix allow-list: - azure @@ -1714,6 +1701,7 @@ repo-azure-key-type: inherit: repo-azure-container + type: string-id default: shared allow-list: - shared @@ -1722,7 +1710,7 @@ repo-azure-uri-style: section: global group: repo - type: string + type: string-id default: host allow-list: - host @@ -1745,7 +1733,7 @@ repo-cipher-type: section: global - type: string + type: string-id group: repo default: none allow-list: @@ -1785,7 +1773,7 @@ repo-gcs-key-type: section: global - type: string + type: string-id group: repo default: service allow-list: @@ -1811,10 +1799,9 @@ hardlink: {} repo-host-type: - internal: true section: global group: repo - type: string + type: string-id default: ssh allow-list: - ssh @@ -1844,7 +1831,6 @@ backup-host: {} repo-host-cert-file: - internal: true section: global group: repo type: string @@ -1984,7 +1970,7 @@ repo-retention-archive-type: section: global - type: string + type: string-id group: repo default: full allow-list: @@ -2016,7 +2002,7 @@ repo-retention-full-type: section: global group: repo - type: string + type: string-id default: count allow-list: - count @@ -2088,6 +2074,7 @@ repo-s3-key-type: inherit: repo-s3-bucket + type: string-id default: shared allow-list: - shared @@ -2115,7 +2102,7 @@ repo-s3-uri-style: section: global group: repo - type: string + type: string-id default: host allow-list: - host diff -Nru pgbackrest-2.36/src/build/config/main.c pgbackrest-2.37/src/build/config/main.c --- pgbackrest-2.36/src/build/config/main.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/config/main.c 2022-01-03 13:43:55.000000000 +0000 @@ -13,7 +13,7 @@ main(int argListSize, const char *argList[]) { // Check parameters - CHECK(argListSize <= 2); + CHECK(ParamInvalidError, argListSize <= 2, "only one parameter allowed"); // Initialize logging logInit(logLevelWarn, logLevelError, logLevelOff, false, 0, 1, false); diff -Nru pgbackrest-2.36/src/build/config/parse.c pgbackrest-2.37/src/build/config/parse.c --- pgbackrest-2.36/src/build/config/parse.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/config/parse.c 2022-01-03 13:43:55.000000000 +0000 @@ -346,11 +346,11 @@ { // Else allow list is inherited - CHECK(optList != NULL); + CHECK(AssertError, optList != NULL, "option list is NULL"); yamlEventCheck(allowListVal, yamlEventTypeScalar); const BldCfgOptionRaw *const optInherit = lstFind(optList, &allowListVal.value); - CHECK(optInherit != NULL); + CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); MEM_CONTEXT_PRIOR_BEGIN() { @@ -359,7 +359,7 @@ MEM_CONTEXT_PRIOR_END(); } - CHECK(result != NULL); + CHECK(AssertError, result != NULL, "result is NULL"); } MEM_CONTEXT_TEMP_END(); @@ -461,7 +461,7 @@ else { // Else depend is inherited - CHECK(optList != NULL); + CHECK(AssertError, optList != NULL, "option list is NULL"); yamlEventCheck(dependVal, yamlEventTypeScalar); const BldCfgOptionRaw *const optInherit = lstFind(optList, &dependVal.value); @@ -543,7 +543,7 @@ MEM_CONTEXT_PRIOR_END(); deprecate = lstFind(result, &name); - CHECK(deprecate != NULL); + CHECK(AssertError, deprecate != NULL, "deprecate is NULL"); } // Set indexed/unindexed flags @@ -700,11 +700,11 @@ // Else command list is inherited else { - CHECK(optList != NULL); + CHECK(AssertError, optList != NULL, "option list is NULL"); yamlEventCheck(optCmdVal, yamlEventTypeScalar); const BldCfgOptionRaw *const optInherit = lstFind(optList, &optCmdVal.value); - CHECK(optInherit != NULL); + CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); result = optInherit->cmdList; } @@ -797,7 +797,7 @@ else if (strEqZ(optDef.value, "inherit")) { const BldCfgOptionRaw *const optInherit = lstFind(optListRaw, &optDefVal.value); - CHECK(optInherit != NULL); + CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); optRaw = *optInherit; optRaw.name = opt.value; @@ -971,7 +971,7 @@ optCmd.roleList = cmd->roleList; } - CHECK(optCmd.roleList != NULL); + CHECK(AssertError, optCmd.roleList != NULL, "role list is NULL"); MEM_CONTEXT_BEGIN(lstMemContext(cmdOptList)) { @@ -992,7 +992,7 @@ } const BldCfgOption *const opt = lstGet(result, optRawIdx); - CHECK(strEq(opt->name, optRaw->name)); + CHECK(AssertError, strEq(opt->name, optRaw->name), "option name does not equal raw option name"); // Assigning to const pointers this way is definitely cheating, but since we allocated the memory and know exactly where // it is so this is the easiest way to avoid a lot of indirection due to the dependency of BldCfgOptionDepend on diff -Nru pgbackrest-2.36/src/build/config/parse.h pgbackrest-2.37/src/build/config/parse.h --- pgbackrest-2.36/src/build/config/parse.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/config/parse.h 2022-01-03 13:43:55.000000000 +0000 @@ -43,7 +43,7 @@ STRING_DECLARE(OPT_TYPE_SIZE_STR); #define OPT_TYPE_STRING "string" STRING_DECLARE(OPT_TYPE_STRING_STR); -#define OPT_TYPE_STRING_ID "stringId" +#define OPT_TYPE_STRING_ID "string-id" STRING_DECLARE(OPT_TYPE_STRING_ID_STR); #define OPT_TYPE_TIME "time" STRING_DECLARE(OPT_TYPE_TIME_STR); diff -Nru pgbackrest-2.36/src/build/config/render.c pgbackrest-2.37/src/build/config/render.c --- pgbackrest-2.36/src/build/config/render.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/config/render.c 2022-01-03 13:43:55.000000000 +0000 @@ -7,6 +7,7 @@ #include "common/log.h" #include "common/type/convert.h" +#include "config/common.h" #include "storage/posix/storage.h" #include "build/common/render.h" @@ -152,7 +153,7 @@ { const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - if (strEqZ(opt->type, CFGDEF_TYPE_STRING)) + if (strEq(opt->type, OPT_TYPE_STRING_ID_STR)) { StringList *const allowList = strLstNew(); @@ -175,28 +176,27 @@ strLstSort(allowList, sortOrderAsc); - if (!strLstEmpty(allowList)) - { - if (lf) - strCatChr(config, '\n'); + ASSERT(!strLstEmpty(allowList)); - for (unsigned int allowListIdx = 0; allowListIdx < strLstSize(allowList); allowListIdx++) - { - const String *const allowListItem = strLstGet(allowList, allowListIdx); - const String *const constPrefix = strUpper( - strReplaceChr(strNewFmt("CFGOPTVAL_%s_%s", strZ(opt->name), strZ(allowListItem)), '-', '_')); + if (lf) + strCatChr(config, '\n'); - // Render StringId - strCatFmt(config, "%s\n", strZ(bldDefineRender(constPrefix, bldStrId(strZ(allowListItem))))); + for (unsigned int allowListIdx = 0; allowListIdx < strLstSize(allowList); allowListIdx++) + { + const String *const allowListItem = strLstGet(allowList, allowListIdx); + const String *const constPrefix = strUpper( + strReplaceChr(strNewFmt("CFGOPTVAL_%s_%s", strZ(opt->name), strZ(allowListItem)), '-', '_')); - // Render Z - strCatFmt( - config, "%s\n", - strZ(bldDefineRender(strNewFmt("%s_Z", strZ(constPrefix)), strNewFmt("\"%s\"", strZ(allowListItem))))); - } + // Render StringId + strCatFmt(config, "%s\n", strZ(bldDefineRender(constPrefix, bldStrId(strZ(allowListItem))))); - lf = true; + // Render Z + strCatFmt( + config, "%s\n", + strZ(bldDefineRender(strNewFmt("%s_Z", strZ(constPrefix)), strNewFmt("\"%s\"", strZ(allowListItem))))); } + + lf = true; } } @@ -384,11 +384,15 @@ if (strEq(optType, OPT_TYPE_TIME_STR)) { - value = (int64_t)(cvtZToDouble(strZ(scalar)) * 1000); + value = cfgParseTime(scalar); + } + else if (strEq(optType, OPT_TYPE_SIZE_STR)) + { + value = cfgParseSize(scalar); } else { - CHECK(strEq(optType, OPT_TYPE_SIZE_STR) || strEq(optType, OPT_TYPE_INTEGER_STR)); + CHECK(AssertError, strEq(optType, OPT_TYPE_SIZE_STR) || strEq(optType, OPT_TYPE_INTEGER_STR), "invalid type"); value = cvtZToInt64(strZ(scalar)); } @@ -491,7 +495,7 @@ " PARSE_RULE_OPTIONAL_DEFAULT\n" " (\n"); - if (!strEq(optType, OPT_TYPE_STRING_STR) && !strEq(optType, OPT_TYPE_PATH_STR) && !strEq(optType, OPT_TYPE_STRING_ID_STR)) + if (!strEq(optType, OPT_TYPE_STRING_STR) && !strEq(optType, OPT_TYPE_PATH_STR)) strCatFmt(result, " %s,\n", strZ(bldCfgRenderScalar(defaultValue, optType))); if (!strEq(optType, OPT_TYPE_BOOLEAN_STR)) @@ -510,6 +514,22 @@ return result; } +// Helper to add values to value lists +static void +bldCfgRenderValueAdd( + const String *const optType, const String *const value, StringList *const ruleDataList, StringList *const ruleStrList) +{ + if (strEq(optType, OPT_TYPE_TIME_STR)) + strLstAddIfMissing(ruleDataList, strNewFmt("%" PRId64, cfgParseTime(value))); + else if (strEq(optType, OPT_TYPE_SIZE_STR)) + strLstAddIfMissing(ruleDataList, strNewFmt("%" PRId64, cfgParseSize(value))); + else + strLstAddIfMissing(ruleDataList, value); + + if (ruleStrList != NULL && !strEq(optType, OPT_TYPE_STRING_STR) && !strEq(optType, OPT_TYPE_PATH_STR)) + strLstAddIfMissing(ruleStrList, strNewFmt("\"%s\"", strZ(value))); +} + static void bldCfgRenderParseAutoC(const Storage *const storageRepo, const BldCfg bldCfg) { @@ -745,16 +765,6 @@ } } - // Determine if the option has an allow list. This will decide whether it is treated as a String or StringId. This should be - // replaced with a StringId type. - bool allowList = opt->allowList != NULL; - - for (unsigned int optCmdIdx = 0; optCmdIdx < lstSize(opt->cmdList); optCmdIdx++) - { - if (((BldCfgOptionCommand *)lstGet(opt->cmdList, optCmdIdx))->allowList != NULL) - allowList = true; - } - // Build default optional rules KeyValue *const optionalDefaultRule = kvNew(); const Variant *const ruleDepend = VARSTRDEF("01-depend"); @@ -773,53 +783,24 @@ optionalDefaultRule, ruleAllowRange, VARSTR(bldCfgRenderAllowRange(opt->allowRangeMin, opt->allowRangeMax, opt->type))); - if (strEq(opt->type, OPT_TYPE_TIME_STR)) - { - strLstAddIfMissing(ruleDataList, strNewFmt("%" PRId64, (int64_t)(cvtZToDouble(strZ(opt->allowRangeMin)) * 1000))); - strLstAddIfMissing(ruleDataList, strNewFmt("%" PRId64, (int64_t)(cvtZToDouble(strZ(opt->allowRangeMax)) * 1000))); - } - else - { - strLstAddIfMissing(ruleDataList, opt->allowRangeMin); - strLstAddIfMissing(ruleDataList, opt->allowRangeMax); - } + bldCfgRenderValueAdd(opt->type, opt->allowRangeMin, ruleDataList, NULL); + bldCfgRenderValueAdd(opt->type, opt->allowRangeMax, ruleDataList, NULL); } if (opt->allowList != NULL) { - kvAdd( - optionalDefaultRule, ruleAllowList, - VARSTR( - bldCfgRenderAllowList( - opt->allowList, strEq(opt->type, OPT_TYPE_STRING_STR) ? OPT_TYPE_STRING_ID_STR : opt->type))); + kvAdd(optionalDefaultRule, ruleAllowList, VARSTR(bldCfgRenderAllowList(opt->allowList, opt->type))); for (unsigned int allowIdx = 0; allowIdx < strLstSize(opt->allowList); allowIdx++) - strLstAddIfMissing(ruleDataList, strLstGet(opt->allowList, allowIdx)); + bldCfgRenderValueAdd(opt->type, strLstGet(opt->allowList, allowIdx), ruleDataList, NULL); } if (opt->defaultValue != NULL) { - kvAdd( - optionalDefaultRule, ruleDefault, - VARSTR( - bldCfgRenderDefault( - opt->defaultValue, opt->defaultLiteral, - strEq(opt->type, OPT_TYPE_STRING_STR) && allowList ? OPT_TYPE_STRING_ID_STR : opt->type))); + kvAdd(optionalDefaultRule, ruleDefault, VARSTR(bldCfgRenderDefault(opt->defaultValue, opt->defaultLiteral, opt->type))); if (!strEq(opt->type, OPT_TYPE_BOOLEAN_STR)) - { - if (strEq(opt->type, OPT_TYPE_TIME_STR)) - { - strLstAddIfMissing( - ruleDataList, strNewFmt("%" PRId64, (int64_t)(cvtZToDouble(strZ(opt->defaultValue)) * 1000))); - } - else - strLstAddIfMissing(ruleDataList, opt->defaultValue); - - strLstAddIfMissing( - ruleStrList, - strNewFmt("%s%s%s", opt->defaultLiteral ? "" : "\"", strZ(opt->defaultValue), opt->defaultLiteral ? "" : "\"")); - } + bldCfgRenderValueAdd(opt->type, opt->defaultValue, ruleDataList, ruleStrList); } // Build command optional rules @@ -837,14 +818,10 @@ // Allow lists if (optCmd->allowList != NULL) { - kvAdd( - optionalCmdRuleType, ruleAllowList, - VARSTR( - bldCfgRenderAllowList( - optCmd->allowList, strEq(opt->type, OPT_TYPE_STRING_STR) ? OPT_TYPE_STRING_ID_STR : opt->type))); + kvAdd(optionalCmdRuleType, ruleAllowList, VARSTR(bldCfgRenderAllowList(optCmd->allowList, opt->type))); for (unsigned int allowIdx = 0; allowIdx < strLstSize(optCmd->allowList); allowIdx++) - strLstAddIfMissing(ruleDataList, strLstGet(optCmd->allowList, allowIdx)); + bldCfgRenderValueAdd(opt->type, strLstGet(optCmd->allowList, allowIdx), ruleDataList, NULL); } // Defaults @@ -852,27 +829,10 @@ { kvAdd( optionalCmdRuleType, ruleDefault, - VARSTR( - bldCfgRenderDefault( - optCmd->defaultValue, opt->defaultLiteral, - strEq(opt->type, OPT_TYPE_STRING_STR) && allowList ? OPT_TYPE_STRING_ID_STR : opt->type))); + VARSTR(bldCfgRenderDefault(optCmd->defaultValue, opt->defaultLiteral, opt->type))); if (!strEq(opt->type, OPT_TYPE_BOOLEAN_STR)) - { - if (strEq(opt->type, OPT_TYPE_TIME_STR)) - { - strLstAddIfMissing( - ruleDataList, strNewFmt("%" PRId64, (int64_t)(cvtZToDouble(strZ(optCmd->defaultValue)) * 1000))); - } - else - strLstAddIfMissing(ruleDataList, optCmd->defaultValue); - - strLstAddIfMissing( - ruleStrList, - strNewFmt( - "%s%s%s", opt->defaultLiteral ? "" : "\"", strZ(optCmd->defaultValue), - opt->defaultLiteral ? "" : "\"")); - } + bldCfgRenderValueAdd(opt->type, optCmd->defaultValue, ruleDataList, ruleStrList); } // Requires @@ -1003,10 +963,11 @@ if (strEq(opt->type, OPT_TYPE_STRING_STR) || strEq(opt->type, OPT_TYPE_PATH_STR)) { - if (allowList) - ruleAddList = ruleStrIdList; - else - ruleAddList = ruleStrList; + ruleAddList = ruleStrList; + } + else if (strEq(opt->type, OPT_TYPE_STRING_ID_STR)) + { + ruleAddList = ruleStrIdList; } else { @@ -1017,19 +978,20 @@ for (unsigned int ruleDataIdx = 0; ruleDataIdx < strLstSize(ruleDataList); ruleDataIdx++) { if (ruleInt) + { strLstAddIfMissing(ruleAddList, strNewFmt("%20s", strZ(strLstGet(ruleDataList, ruleDataIdx)))); + } + else if (strEq(opt->type, OPT_TYPE_STRING_ID_STR)) + { + strLstAddIfMissing(ruleAddList, strLstGet(ruleDataList, ruleDataIdx)); + } else { - if (allowList) - strLstAddIfMissing(ruleAddList, strLstGet(ruleDataList, ruleDataIdx)); - else - { - strLstAddIfMissing( - ruleAddList, - strNewFmt( - "%s%s%s", opt->defaultLiteral ? "" : "\"", strZ(strLstGet(ruleDataList, ruleDataIdx)), - opt->defaultLiteral ? "" : "\"")); - } + strLstAddIfMissing( + ruleAddList, + strNewFmt( + "%s%s%s", opt->defaultLiteral ? "" : "\"", strZ(strLstGet(ruleDataList, ruleDataIdx)), + opt->defaultLiteral ? "" : "\"")); } } } diff -Nru pgbackrest-2.36/src/build/configure.ac pgbackrest-2.37/src/build/configure.ac --- pgbackrest-2.36/src/build/configure.ac 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/configure.ac 2022-01-03 13:43:55.000000000 +0000 @@ -1,7 +1,7 @@ # Initialize configuration # ---------------------------------------------------------------------------------------------------------------------------------- AC_PREREQ([2.69]) -AC_INIT([pgBackRest], [2.36]) +AC_INIT([pgBackRest], [2.37]) AC_CONFIG_SRCDIR([version.h]) AC_CONFIG_AUX_DIR(build) diff -Nru pgbackrest-2.36/src/build/error/main.c pgbackrest-2.37/src/build/error/main.c --- pgbackrest-2.36/src/build/error/main.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/error/main.c 2022-01-03 13:43:55.000000000 +0000 @@ -13,7 +13,7 @@ main(int argListSize, const char *argList[]) { // Check parameters - CHECK(argListSize <= 2); + CHECK(ParamInvalidError, argListSize <= 2, "only one parameter allowed"); // Initialize logging logInit(logLevelWarn, logLevelError, logLevelOff, false, 0, 1, false); diff -Nru pgbackrest-2.36/src/build/help/help.xml pgbackrest-2.37/src/build/help/help.xml --- pgbackrest-2.36/src/build/help/help.xml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/help/help.xml 2022-01-03 13:43:55.000000000 +0000 @@ -132,12 +132,26 @@

Buffer size used for copy, compress, encrypt, and other operations. The number of buffers used depends on options and each operation may use additional memory, e.g. gz compression may use an additional 256KiB of memory.

-

Size can be entered in bytes (default) or KB, MB, GB, TB, or PB where the multiplier is a power of 1024. For example, the case-insensitive value 32k (or 32KB) can be used instead of 32768.

+ +

Size can be entered in bytes (default) or KiB, MiB, GiB, TiB, or PiB where the multiplier is a power of 1024. For example, the case-insensitive value 5GiB (or 5GB, 5g) can be used instead of 5368709120. Fractional values such as 2.5GiB are not allowed, use 2560MiB instead.

-

Allowed values, in bytes, are 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, and 16777216.

+

Allowed values are 16KiB, 32KiB, 64KiB, 128KiB, 256KiB, 512KiB, 1MiB, 2MiB, 4MiB, 8MiB, and 16MiB.

 - 32K + 2MiB + + + + + command. + + +

may generate a command string, e.g. when the restore command generates the restore_command setting. The command used to run the process will be used in this case unless the cmd option is provided.

+ + Wrapping the command may cause unpredictable behavior and is not recommended. + + + /var/lib/pgsql/bin/pgbackrest_wrapper.sh @@ -1125,10 +1139,11 @@

Defines how often the manifest will be saved during a backup. Saving the manifest is important because it stores the checksums and allows the resume function to work efficiently. The actual threshold used is 1% of the backup size or manifest-save-threshold, whichever is greater.

-

Size can be entered in bytes (default) or KB, MB, GB, TB, or PB where the multiplier is a power of 1024.

+ +

Size can be entered in bytes (default) or KiB, MiB, GiB, TiB, or PiB where the multiplier is a power of 1024. For example, the case-insensitive value 5GiB (or 5GB, 5g) can be used instead of 5368709120. Fractional values such as 2.5GiB are not allowed, use 2560MiB instead.

 - 5G + 8GiB @@ -1212,10 +1227,11 @@

Specifies the maximum size of the archive-get queue when archive-async is enabled. The queue is stored in the spool-path and is used to speed providing WAL to .

-

Size can be entered in bytes (default) or KB, MB, GB, TB, or PB where the multiplier is a power of 1024.

+ +

Size can be entered in bytes (default) or KiB, MiB, GiB, TiB, or PiB where the multiplier is a power of 1024. For example, the case-insensitive value 5GiB (or 5GB, 5g) can be used instead of 5368709120. Fractional values such as 2.5GiB are not allowed, use 2560MiB instead.

 - 1073741824 + 1GiB @@ -1249,10 +1265,11 @@

The purpose of this feature is to prevent the log volume from filling up at which point Postgres will stop completely. Better to lose the backup than have go down.

-

Size can be entered in bytes (default) or KB, MB, GB, TB, or PB where the multiplier is a power of 1024.

+ +

Size can be entered in bytes (default) or KiB, MiB, GiB, TiB, or PiB where the multiplier is a power of 1024. For example, the case-insensitive value 5GiB (or 5GB, 5g) can be used instead of 5368709120. Fractional values such as 2.5GiB are not allowed, use 2560MiB instead.

- 1GB + 1TiB @@ -1380,15 +1397,23 @@ Restore all tablespaces into the specified directory. -

By default tablespaces are restored into their original locations and while this behavior can be modified by with the tablespace-map open it is sometime preferable to remap all tablespaces to a new directory all at once. This is particularly useful for development or staging systems that may not have the same storage layout as the original system where the backup was generated.

+

Tablespaces are restored into their original locations by default. This behavior can be modified for each tablespace with the tablespace-map option, but it is sometimes preferable to remap all tablespaces to a new directory all at once. This is particularly useful for development or staging systems that may not have the same storage layout as the original system where the backup was generated.

The path specified will be the parent path used to create all the tablespaces in the backup.

 /data/tablespace + + - + + + +

The server section defines options used for configuring the TLS server.

+ + + TLS server authorized clients. @@ -2256,8 +2281,8 @@ - - Start server. + + server.

The server allows access to remote hosts without using the SSH protocol.

diff -Nru pgbackrest-2.36/src/build/help/main.c pgbackrest-2.37/src/build/help/main.c --- pgbackrest-2.36/src/build/help/main.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/help/main.c 2022-01-03 13:43:55.000000000 +0000 @@ -14,7 +14,7 @@ main(int argListSize, const char *argList[]) { // Check parameters - CHECK(argListSize <= 2); + CHECK(ParamInvalidError, argListSize <= 2, "only one parameter allowed"); // Initialize logging logInit(logLevelWarn, logLevelError, logLevelOff, false, 0, 1, false); diff -Nru pgbackrest-2.36/src/build/help/parse.c pgbackrest-2.37/src/build/help/parse.c --- pgbackrest-2.36/src/build/help/parse.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/help/parse.c 2022-01-03 13:43:55.000000000 +0000 @@ -154,7 +154,7 @@ { const BldCfgOptionCommand *const optCmd = lstGet(opt->cmdList, optCmdListIdx); const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &optCmd->name); - CHECK(cmdHlp != NULL); + CHECK(AssertError, cmdHlp != NULL, "command help is NULL"); // Only options with a command role of main require help if (!strLstExists(optCmd->roleList, CMD_ROLE_MAIN_STR)) diff -Nru pgbackrest-2.36/src/build/help/render.c pgbackrest-2.37/src/build/help/render.c --- pgbackrest-2.36/src/build/help/render.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/build/help/render.c 2022-01-03 13:43:55.000000000 +0000 @@ -133,7 +133,7 @@ { const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &cmd->name); - CHECK(cmdHlp != NULL); + CHECK(AssertError, cmdHlp != NULL, "command help is NULL"); pckWriteBoolP(pack, cmd->internal); pckWriteStrP(pack, bldHlpRenderXml(cmdHlp->summary)); @@ -209,7 +209,7 @@ if (optCmd != NULL) { const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &cmd->name); - CHECK(cmdHlp != NULL); + CHECK(AssertError, cmdHlp != NULL, "command help is NULL"); const BldHlpOption *const cmdOptHlp = cmdHlp->optList != NULL ? lstFind(cmdHlp->optList, &opt->name) : NULL; if (opt->internal != optCmd->internal || cmdOptHlp != NULL) diff -Nru pgbackrest-2.36/src/command/archive/common.c pgbackrest-2.37/src/command/archive/common.c --- pgbackrest-2.36/src/command/archive/common.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/archive/common.c 2022-01-03 13:43:55.000000000 +0000 @@ -17,6 +17,7 @@ #include "common/regExp.h" #include "common/wait.h" #include "config/config.h" +#include "postgres/interface.h" #include "postgres/version.h" #include "storage/helper.h" #include "storage/helper.h" @@ -274,7 +275,7 @@ THROW_ON_SYS_ERROR(waitpid(pid, &processStatus, 0) == -1, ExecuteError, "unable to wait for forked process"); // The first fork should exit with success. If not, something went wrong during the second fork. - CHECK(WIFEXITED(processStatus) && WEXITSTATUS(processStatus) == 0); + CHECK(ExecuteError, WIFEXITED(processStatus) && WEXITSTATUS(processStatus) == 0, "error on first fork"); #ifdef DEBUG_EXEC_TIME // If the process does not exit immediately then something probably went wrong with the double fork. It's possible that this @@ -492,7 +493,7 @@ MEM_CONTEXT_TEMP_BEGIN() { - timeline = (uint32_t)strtol(strZ(strSubN(walSegment, 0, 8)), NULL, 16); + timeline = pgTimelineFromWalSegment(walSegment); major = (uint32_t)strtol(strZ(strSubN(walSegment, 8, 8)), NULL, 16); minor = (uint32_t)strtol(strZ(strSubN(walSegment, 16, 8)), NULL, 16); diff -Nru pgbackrest-2.36/src/command/archive/push/push.c pgbackrest-2.37/src/command/archive/push/push.c --- pgbackrest-2.36/src/command/archive/push/push.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/archive/push/push.c 2022-01-03 13:43:55.000000000 +0000 @@ -413,7 +413,7 @@ // Push the file to the archive ArchivePushFileResult fileResult = archivePushFile( walFile, cfgOptionBool(cfgOptArchiveHeaderCheck), archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile, - compressTypeEnum(cfgOptionStr(cfgOptCompressType)), cfgOptionInt(cfgOptCompressLevel), archiveInfo.repoList, + compressTypeEnum(cfgOptionStrId(cfgOptCompressType)), cfgOptionInt(cfgOptCompressLevel), archiveInfo.repoList, archiveInfo.errorList); // If a warning was returned then log it @@ -526,7 +526,7 @@ ArchivePushAsyncData jobData = { .walPath = strLstGet(commandParam, 0), - .compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)), + .compressType = compressTypeEnum(cfgOptionStrId(cfgOptCompressType)), .compressLevel = cfgOptionInt(cfgOptCompressLevel), }; diff -Nru pgbackrest-2.36/src/command/backup/backup.c pgbackrest-2.37/src/command/backup/backup.c --- pgbackrest-2.36/src/command/backup/backup.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/backup/backup.c 2022-01-03 13:43:55.000000000 +0000 @@ -148,6 +148,10 @@ const Storage *storageStandby; // Storage object for the standby const String *hostStandby; // Host name of the standby + const InfoArchive *archiveInfo; // Archive info + const String *archiveId; // Archive where backup WAL will be stored + + unsigned int timeline; // Primary timeline unsigned int version; // PostgreSQL version unsigned int walSegmentSize; // PostgreSQL wal segment size } BackupData; @@ -186,6 +190,8 @@ } // Get database info when online + PgControl pgControl = {0}; + if (cfgOptionBool(cfgOptOnline)) { bool backupStandby = cfgOptionBool(cfgOptBackupStandby); @@ -203,15 +209,19 @@ result->storageStandby = storagePgIdx(result->pgIdxStandby); result->hostStandby = cfgOptionIdxStrNull(cfgOptPgHost, result->pgIdxStandby); } + + // Get pg_control info from the primary + pgControl = dbPgControl(result->dbPrimary); } + // Else get pg_control info directly from the file + else + pgControl = pgControlFromFile(storagePgIdx(result->pgIdxPrimary)); // Add primary info result->storagePrimary = storagePgIdx(result->pgIdxPrimary); result->hostPrimary = cfgOptionIdxStrNull(cfgOptPgHost, result->pgIdxPrimary); - // Get pg_control info from the primary - PgControl pgControl = pgControlFromFile(result->storagePrimary); - + result->timeline = pgControl.timeline; result->version = pgControl.version; result->walSegmentSize = pgControl.walSegmentSize; @@ -259,6 +269,15 @@ cfgOptionSet(cfgOptChecksumPage, cfgSourceParam, BOOL_FALSE_VAR); } + // Get archive info + if (cfgOptionBool(cfgOptOnline) && cfgOptionBool(cfgOptArchiveCheck)) + { + result->archiveInfo = infoArchiveLoadFile( + storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cfgOptionStrId(cfgOptRepoCipherType), + cfgOptionStrNull(cfgOptRepoCipherPass)); + result->archiveId = infoArchiveId(result->archiveInfo); + } + FUNCTION_LOG_RETURN(BACKUP_DATA, result); } @@ -368,7 +387,7 @@ strZ(manifestData(result)->backrestVersion)); // Warn if compress-type option changed - if (compressTypeEnum(cfgOptionStr(cfgOptCompressType)) != manifestPriorData->backupOptionCompressType) + if (compressTypeEnum(cfgOptionStrId(cfgOptCompressType)) != manifestPriorData->backupOptionCompressType) { LOG_WARN_FMT( "%s backup cannot alter " CFGOPT_COMPRESS_TYPE " option to '%s', reset to value in %s", @@ -381,9 +400,11 @@ // There's a small chance that the prior manifest is old enough that backupOptionCompressLevel was not recorded. // There's an even smaller chance that the user will also alter compression-type in this scenario right after - // upgrading to a newer version. Because we judge this combination of events to be nearly impossible just assert + // upgrading to a newer version. Because we judge this combination of events to be nearly impossible just check // here so no test coverage is needed. - CHECK(manifestPriorData->backupOptionCompressLevel != NULL); + CHECK( + FormatError, manifestPriorData->backupOptionCompressLevel != NULL, + "compress-level missing in prior manifest"); // Set the compression level back to whatever was in the prior backup cfgOptionSet( @@ -430,7 +451,7 @@ else { LOG_WARN_FMT("no prior backup exists, %s backup has been changed to full", strZ(cfgOptionDisplay(cfgOptType))); - cfgOptionSet(cfgOptType, cfgSourceParam, VARSTR(strIdToStr(backupTypeFull))); + cfgOptionSet(cfgOptType, cfgSourceParam, VARUINT64(backupTypeFull)); } } MEM_CONTEXT_TEMP_END(); @@ -666,12 +687,19 @@ // occurs then the backup will be considered unusable and a resume will not be attempted. if (cfgOptionBool(cfgOptResume)) { - reason = strNewFmt("unable to read %s" INFO_COPY_EXT, strZ(manifestFile)); - TRY_BEGIN() { manifestResume = manifestLoadFile( storageRepo(), manifestFile, cfgOptionStrId(cfgOptRepoCipherType), cipherPassBackup); + } + CATCH_ANY() + { + reason = strNewFmt("unable to read %s" INFO_COPY_EXT, strZ(manifestFile)); + } + TRY_END(); + + if (manifestResume != NULL) + { const ManifestData *manifestResumeData = manifestData(manifestResume); // Check pgBackRest version. This allows the resume implementation to be changed with each version of @@ -701,7 +729,8 @@ } // Check compression. Compression can't be changed between backups so resume won't work either. else if ( - manifestResumeData->backupOptionCompressType != compressTypeEnum(cfgOptionStr(cfgOptCompressType))) + manifestResumeData->backupOptionCompressType != + compressTypeEnum(cfgOptionStrId(cfgOptCompressType))) { reason = strNewFmt( "new compression '%s' does not match resumable compression '%s'", @@ -711,10 +740,6 @@ else usable = true; } - CATCH_ANY() - { - } - TRY_END(); } } @@ -777,7 +802,7 @@ { .manifest = manifest, .manifestResume = manifestResume, - .compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)), + .compressType = compressTypeEnum(cfgOptionStrId(cfgOptCompressType)), .delta = cfgOptionBool(cfgOptDelta), .backupPath = strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(manifestData(manifest)->backupLabel)), }; @@ -847,7 +872,8 @@ cfgOptionBool(cfgOptStartFast) ? "requested immediate" : "next regular"); DbBackupStartResult dbBackupStartResult = dbBackupStart( - backupData->dbPrimary, cfgOptionBool(cfgOptStartFast), cfgOptionBool(cfgOptStopAuto)); + backupData->dbPrimary, cfgOptionBool(cfgOptStartFast), cfgOptionBool(cfgOptStopAuto), + cfgOptionBool(cfgOptArchiveCheck)); MEM_CONTEXT_PRIOR_BEGIN() { @@ -864,14 +890,23 @@ if (cfgOptionBool(cfgOptBackupStandby)) { LOG_INFO_FMT("wait for replay on the standby to reach %s", strZ(result.lsn)); - dbReplayWait(backupData->dbStandby, result.lsn, cfgOptionUInt64(cfgOptArchiveTimeout)); + dbReplayWait(backupData->dbStandby, result.lsn, backupData->timeline, cfgOptionUInt64(cfgOptArchiveTimeout)); LOG_INFO_FMT("replay on the standby reached %s", strZ(result.lsn)); + } - // The standby db object won't be used anymore so free it - dbFree(backupData->dbStandby); - - // The standby protocol connection won't be used anymore so free it - protocolRemoteFree(backupData->pgIdxStandby); + // Check that WAL segments are being archived. If archiving is not working then the backup will eventually fail so + // better to catch it as early as possible. A segment to check may not be available on older versions of PostgreSQL or + // if archive-check is false. + if (dbBackupStartResult.walSegmentCheck != NULL) + { + LOG_INFO_FMT( + "check archive for %ssegment %s", + strEq(result.walSegmentName, dbBackupStartResult.walSegmentCheck) ? "" : "prior ", + strZ(dbBackupStartResult.walSegmentCheck)); + + walSegmentFind( + storageRepo(), backupData->archiveId, dbBackupStartResult.walSegmentCheck, + cfgOptionUInt64(cfgOptArchiveTimeout)); } } } @@ -902,7 +937,7 @@ { // Create file const String *manifestName = strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strZ(name)); - CompressType compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)); + CompressType compressType = compressTypeEnum(cfgOptionStrId(cfgOptCompressType)); StorageWrite *write = storageNewWriteP( storageRepoWrite(), @@ -1098,7 +1133,7 @@ } // Check that the array was not empty - CHECK(first); + CHECK(FormatError, first, "page checksum result array is empty"); // Output last page or page range backupJobResultPageChecksumOut(result, pageBegin, pageEnd); @@ -1112,10 +1147,10 @@ /*********************************************************************************************************************************** Log the results of a job and throw errors ***********************************************************************************************************************************/ -static uint64_t +static void backupJobResult( Manifest *manifest, const String *host, const String *const fileName, StringList *fileRemove, ProtocolParallelJob *const job, - const uint64_t sizeTotal, uint64_t sizeCopied) + const uint64_t sizeTotal, uint64_t *sizeProgress) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(MANIFEST, manifest); @@ -1124,7 +1159,7 @@ FUNCTION_LOG_PARAM(STRING_LIST, fileRemove); FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, job); FUNCTION_LOG_PARAM(UINT64, sizeTotal); - FUNCTION_LOG_PARAM(UINT64, sizeCopied); + FUNCTION_LOG_PARAM_P(UINT64, sizeProgress); FUNCTION_LOG_END(); ASSERT(manifest != NULL); @@ -1148,7 +1183,7 @@ PackRead *const checksumPageResult = pckReadPackReadP(jobResult); // Increment backup copy progress - sizeCopied += copySize; + *sizeProgress += copySize; // Create log file name const String *fileLog = host == NULL ? fileName : strNewFmt("%s:%s", strZ(host), strZ(fileName)); @@ -1156,7 +1191,7 @@ // Format log strings const String *const logProgress = strNewFmt( - "%s, %" PRIu64 "%%", strZ(strSizeFormat(copySize)), sizeTotal == 0 ? 100 : sizeCopied * 100 / sizeTotal); + "%s, %" PRIu64 "%%", strZ(strSizeFormat(copySize)), sizeTotal == 0 ? 100 : *sizeProgress * 100 / sizeTotal); const String *const logChecksum = copySize != 0 ? strNewFmt(" checksum %s", strZ(copyChecksum)) : EMPTY_STR; // If the file is in a prior backup and nothing changed, just log it @@ -1224,8 +1259,8 @@ else { // Format the page checksum errors - CHECK(checksumPageErrorList != NULL); - CHECK(!varLstEmpty(checksumPageErrorList)); + CHECK(FormatError, checksumPageErrorList != NULL, "page checksum error list is missing"); + CHECK(FormatError, !varLstEmpty(checksumPageErrorList), "page checksum error list is empty"); String *error = strNew(); unsigned int errorTotalMin = 0; @@ -1285,7 +1320,7 @@ else THROW_CODE(protocolParallelJobErrorCode(job), strZ(protocolParallelJobErrorMessage(job))); - FUNCTION_LOG_RETURN(UINT64, sizeCopied); + FUNCTION_LOG_RETURN_VOID(); } /*********************************************************************************************************************************** @@ -1323,6 +1358,30 @@ } /*********************************************************************************************************************************** +Check that the clusters are alive and correctly configured during the backup +***********************************************************************************************************************************/ +static void +backupDbPing(const BackupData *const backupData, const bool force) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(BACKUP_DATA, backupData); + FUNCTION_LOG_PARAM(BOOL, force); + FUNCTION_LOG_END(); + + ASSERT(backupData != NULL); + + if (cfgOptionBool(cfgOptOnline)) + { + dbPing(backupData->dbPrimary, force); + + if (cfgOptionBool(cfgOptBackupStandby)) + dbPing(backupData->dbStandby, force); + } + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** Process the backup manifest ***********************************************************************************************************************************/ // Comparator to order ManifestFile objects by size then name @@ -1362,8 +1421,8 @@ MEM_CONTEXT_TEMP_BEGIN() { - // Create list of process queue - *queueList = lstNewP(sizeof(List *)); + // Create list of process queues (use void * instead of List * to avoid Coverity false positive) + *queueList = lstNewP(sizeof(void *)); // Generate the list of targets StringList *targetList = strLstNew(); @@ -1421,8 +1480,7 @@ do { - // A target should always be found - CHECK(targetIdx < strLstSize(targetList)); + CHECK(AssertError, targetIdx < strLstSize(targetList), "backup target not found"); if (strBeginsWith(file->name, strLstGet(targetList, targetIdx))) break; @@ -1581,7 +1639,7 @@ FUNCTION_TEST_RETURN(result); } -static uint64_t +static void backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart, const String *cipherPassBackup) { FUNCTION_LOG_BEGIN(logLevelDebug); @@ -1648,7 +1706,7 @@ { .backupLabel = backupLabel, .backupStandby = backupStandby, - .compressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)), + .compressType = compressTypeEnum(cfgOptionStrId(cfgOptCompressType)), .compressLevel = cfgOptionInt(cfgOptCompressLevel), .cipherType = cfgOptionStrId(cfgOptRepoCipherType), .cipherSubPass = manifestCipherSubPass(manifest), @@ -1684,7 +1742,7 @@ manifestSaveSize = cfgOptionUInt64(cfgOptManifestSaveThreshold); // Process jobs - uint64_t sizeCopied = 0; + uint64_t sizeProgress = 0; MEM_CONTEXT_TEMP_RESET_BEGIN() { @@ -1696,23 +1754,26 @@ { ProtocolParallelJob *job = protocolParallelResult(parallelExec); - sizeCopied = backupJobResult( + backupJobResult( manifest, backupStandby && protocolParallelJobProcessId(job) > 1 ? backupData->hostStandby : backupData->hostPrimary, storagePathP( protocolParallelJobProcessId(job) > 1 ? storagePgIdx(pgIdx) : backupData->storagePrimary, - manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), - fileRemove, job, sizeTotal, sizeCopied); + manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), fileRemove, job, + sizeTotal, &sizeProgress); } // A keep-alive is required here for the remote holding open the backup connection protocolKeepAlive(); + // Check that the clusters are alive and correctly configured during the backup + backupDbPing(backupData, false); + // Save the manifest periodically to preserve checksums for resume - if (sizeCopied - manifestSaveLast >= manifestSaveSize) + if (sizeProgress - manifestSaveLast >= manifestSaveSize) { backupManifestSaveCopy(manifest, cipherPassBackup); - manifestSaveLast = sizeCopied; + manifestSaveLast = sizeProgress; } // Reset the memory context occasionally so we don't use too much memory or slow down processing @@ -1780,18 +1841,18 @@ } MEM_CONTEXT_TEMP_END(); - FUNCTION_LOG_RETURN(UINT64, sizeTotal); + FUNCTION_LOG_RETURN_VOID(); } /*********************************************************************************************************************************** Check and copy WAL segments required to make the backup consistent ***********************************************************************************************************************************/ static void -backupArchiveCheckCopy(Manifest *manifest, unsigned int walSegmentSize, const String *cipherPassBackup) +backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manifest, const String *const cipherPassBackup) { FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(BACKUP_DATA, backupData); FUNCTION_LOG_PARAM(MANIFEST, manifest); - FUNCTION_LOG_PARAM(UINT, walSegmentSize); FUNCTION_TEST_PARAM(STRING, cipherPassBackup); FUNCTION_LOG_END(); @@ -1804,13 +1865,13 @@ { MEM_CONTEXT_TEMP_BEGIN() { - unsigned int timeline = cvtZToUIntBase(strZ(strSubN(manifestData(manifest)->archiveStart, 0, 8)), 16); uint64_t lsnStart = pgLsnFromStr(manifestData(manifest)->lsnStart); uint64_t lsnStop = pgLsnFromStr(manifestData(manifest)->lsnStop); LOG_INFO_FMT( - "check archive for segment(s) %s:%s", strZ(pgLsnToWalSegment(timeline, lsnStart, walSegmentSize)), - strZ(pgLsnToWalSegment(timeline, lsnStop, walSegmentSize))); + "check archive for segment(s) %s:%s", + strZ(pgLsnToWalSegment(backupData->timeline, lsnStart, backupData->walSegmentSize)), + strZ(pgLsnToWalSegment(backupData->timeline, lsnStop, backupData->walSegmentSize))); // Save the backup manifest before getting archive logs in case of failure backupManifestSaveCopy(manifest, cipherPassBackup); @@ -1819,13 +1880,8 @@ const ManifestPath *basePath = manifestPathFind(manifest, MANIFEST_TARGET_PGDATA_STR); // Loop through all the segments in the lsn range - InfoArchive *infoArchive = infoArchiveLoadFile( - storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cfgOptionStrId(cfgOptRepoCipherType), - cfgOptionStrNull(cfgOptRepoCipherPass)); - const String *archiveId = infoArchiveId(infoArchive); - StringList *walSegmentList = pgLsnRangeToWalSegmentList( - manifestData(manifest)->pgVersion, timeline, lsnStart, lsnStop, walSegmentSize); + manifestData(manifest)->pgVersion, backupData->timeline, lsnStart, lsnStop, backupData->walSegmentSize); for (unsigned int walSegmentIdx = 0; walSegmentIdx < strLstSize(walSegmentList); walSegmentIdx++) { @@ -1835,7 +1891,7 @@ // Find the actual wal segment file in the archive const String *archiveFile = walSegmentFind( - storageRepo(), archiveId, walSegment, cfgOptionUInt64(cfgOptArchiveTimeout)); + storageRepo(), backupData->archiveId, walSegment, cfgOptionUInt64(cfgOptArchiveTimeout)); if (cfgOptionBool(cfgOptArchiveCopy)) { @@ -1844,17 +1900,18 @@ // Get compression type of the WAL segment and backup CompressType archiveCompressType = compressTypeFromName(archiveFile); - CompressType backupCompressType = compressTypeEnum(cfgOptionStr(cfgOptCompressType)); + CompressType backupCompressType = compressTypeEnum(cfgOptionStrId(cfgOptCompressType)); // Open the archive file StorageRead *read = storageNewReadP( - storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(archiveId), strZ(archiveFile))); + storageRepo(), + strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strZ(backupData->archiveId), strZ(archiveFile))); IoFilterGroup *filterGroup = ioReadFilterGroup(storageReadIo(read)); // Decrypt with archive key if encrypted cipherBlockFilterGroupAdd( filterGroup, cfgOptionStrId(cfgOptRepoCipherType), cipherModeDecrypt, - infoArchiveCipherPass(infoArchive)); + infoArchiveCipherPass(backupData->archiveInfo)); // Compress/decompress if archive and backup do not have the same compression settings if (archiveCompressType != backupCompressType) @@ -1886,7 +1943,7 @@ storageRepoWrite(), strNewFmt( STORAGE_REPO_BACKUP "/%s/%s%s", strZ(manifestData(manifest)->backupLabel), strZ(manifestName), - strZ(compressExtStr(compressTypeEnum(cfgOptionStr(cfgOptCompressType))))))); + strZ(compressExtStr(compressTypeEnum(cfgOptionStrId(cfgOptCompressType))))))); // Add to manifest ManifestFile file = @@ -1896,7 +1953,7 @@ .mode = basePath->mode & (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH), .user = basePath->user, .group = basePath->group, - .size = walSegmentSize, + .size = backupData->walSegmentSize, .sizeRepo = pckReadU64P(ioFilterGroupResultP(filterGroup, SIZE_FILTER_TYPE)), .timestamp = manifestData(manifest)->backupTimestampStop, }; @@ -2041,7 +2098,8 @@ // Validate the manifest using the copy start time manifestBuildValidate( - manifest, cfgOptionBool(cfgOptDelta), backupTime(backupData, true), compressTypeEnum(cfgOptionStr(cfgOptCompressType))); + manifest, cfgOptionBool(cfgOptDelta), backupTime(backupData, true), + compressTypeEnum(cfgOptionStrId(cfgOptCompressType))); // Build an incremental backup if type is not full (manifestPrior will be freed in this call) if (!backupBuildIncr(infoBackup, manifest, manifestPrior, backupStartResult.walSegmentName)) @@ -2064,7 +2122,17 @@ backupManifestSaveCopy(manifest, cipherPassBackup); // Process the backup manifest - uint64_t backupSizeTotal = backupProcess(backupData, manifest, backupStartResult.lsn, cipherPassBackup); + backupProcess(backupData, manifest, backupStartResult.lsn, cipherPassBackup); + + // Check that the clusters are alive and correctly configured after the backup + backupDbPing(backupData, true); + + // The standby db object and protocol won't be used anymore so free them + if (cfgOptionBool(cfgOptBackupStandby)) + { + dbFree(backupData->dbStandby); + protocolRemoteFree(backupData->pgIdxStandby); + } // Stop the backup BackupStopResult backupStopResult = backupStop(backupData, manifest); @@ -2082,7 +2150,7 @@ dbFree(backupData->dbPrimary); // Check and copy WAL segments required to make the backup consistent - backupArchiveCheckCopy(manifest, backupData->walSegmentSize, cipherPassBackup); + backupArchiveCheckCopy(backupData, manifest, cipherPassBackup); // The primary protocol connection won't be used anymore so free it. This needs to happen after backupArchiveCheckCopy() so // the backup lock is held on the remote which allows conditional archiving based on the backup lock. Any further access to @@ -2096,7 +2164,8 @@ // Backup info LOG_INFO_FMT( "%s backup size = %s, file total = %u", strZ(strIdToStr(manifestData(manifest)->backupType)), - strZ(strSizeFormat(backupSizeTotal)), manifestFileTotal(manifest)); + strZ(strSizeFormat(infoBackupDataByLabel(infoBackup, manifestData(manifest)->backupLabel)->backupInfoSizeDelta)), + manifestFileTotal(manifest)); } MEM_CONTEXT_TEMP_END(); diff -Nru pgbackrest-2.36/src/command/check/check.c pgbackrest-2.37/src/command/check/check.c --- pgbackrest-2.36/src/command/check/check.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/check/check.c 2022-01-03 13:43:55.000000000 +0000 @@ -72,11 +72,8 @@ THROW(ConfigError, "primary database not found\nHINT: check indexed pg-path/pg-host configurations"); } - // Validate the standby database config - PgControl pgControl = pgControlFromFile(storagePgIdx(dbGroup.standbyIdx)); - // Check the user configured path and version against the database - checkDbConfig(pgControl.version, dbGroup.standbyIdx, dbGroup.standby, true); + checkDbConfig(dbPgControl(dbGroup.standby).version, dbGroup.standbyIdx, dbGroup.standby, true); // Check each repository configured for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++) @@ -88,8 +85,8 @@ // Check that the backup and archive info files exist and are valid for the current database of the stanza checkStanzaInfoPg( - storageRepo, pgControl.version, pgControl.systemId, cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx)); + storageRepo, dbPgControl(dbGroup.standby).version, dbPgControl(dbGroup.standby).systemId, + cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx)); } LOG_INFO("switch wal not performed because this is a standby"); @@ -116,11 +113,8 @@ // If a primary is defined, check the configuration and perform a WAL switch and make sure the WAL is archived if (dbGroup.primary != NULL) { - // Validate the primary database config - PgControl pgControl = pgControlFromFile(storagePgIdx(dbGroup.primaryIdx)); - // Check the user configured path and version against the database - checkDbConfig(pgControl.version, dbGroup.primaryIdx, dbGroup.primary, false); + checkDbConfig(dbPgControl(dbGroup.primary).version, dbGroup.primaryIdx, dbGroup.primary, false); // Check configuration of each repo const String **repoArchiveId = memNew(sizeof(String *) * cfgOptionGroupIdxTotal(cfgOptGrpRepo)); @@ -134,8 +128,8 @@ // Check that the backup and archive info files exist and are valid for the current database of the stanza checkStanzaInfoPg( - storageRepo, pgControl.version, pgControl.systemId, cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx)); + storageRepo, dbPgControl(dbGroup.primary).version, dbPgControl(dbGroup.primary).systemId, + cfgOptionIdxStrId(cfgOptRepoCipherType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx)); // Attempt to load the archive info file and retrieve the archiveId InfoArchive *archiveInfo = infoArchiveLoadFile( diff -Nru pgbackrest-2.36/src/command/expire/expire.c pgbackrest-2.37/src/command/expire/expire.c --- pgbackrest-2.36/src/command/expire/expire.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/expire/expire.c 2022-01-03 13:43:55.000000000 +0000 @@ -133,7 +133,7 @@ StringList *backupExpired = expireBackup(infoBackup, backupLabel, repoIdx); // If the latest backup was removed, then update the latest link if not a dry-run - if (infoBackupDataByLabel(infoBackup, latestBackup) == NULL) + if (!infoBackupLabelExists(infoBackup, latestBackup)) { // If retention settings have been configured, then there may be holes in the archives. For example, if the archive // for db-id=1 has 01,02,03,04,05 and F1 backup has archive start-stop 02-03 and retention-full=1 @@ -835,7 +835,7 @@ infoPgCipherPass(infoBackupPg(infoBackup))); // If the ancestor of the resumable backup still exists in backup.info then do not remove the resumable backup - if (infoBackupDataByLabel(infoBackup, manifestData(manifestResume)->backupLabelPrior) != NULL) + if (infoBackupLabelExists(infoBackup, manifestData(manifestResume)->backupLabelPrior)) backupIdx = 1; } } @@ -1020,7 +1020,7 @@ // If a backupLabel was set, then attempt to expire the requested backup if (adhocBackupLabel != NULL) { - if (infoBackupDataByLabel(infoBackup, adhocBackupLabel) != NULL) + if (infoBackupLabelExists(infoBackup, adhocBackupLabel)) { adhocBackupFound = true; expireAdhocBackup(infoBackup, adhocBackupLabel, repoIdx); diff -Nru pgbackrest-2.36/src/command/help/help.c pgbackrest-2.37/src/command/help/help.c --- pgbackrest-2.36/src/command/help/help.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/help/help.c 2022-01-03 13:43:55.000000000 +0000 @@ -150,61 +150,65 @@ Helper function for helpRender() to output values as strings ***********************************************************************************************************************************/ static const String * -helpRenderValue(const Variant *value, ConfigOptionType type) +helpRenderValue(const ConfigOption optionId, const unsigned int optionIdx) { FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(VARIANT, value); - FUNCTION_LOG_PARAM(ENUM, type); + FUNCTION_LOG_PARAM(ENUM, optionId); + FUNCTION_LOG_PARAM(UINT, optionIdx); FUNCTION_LOG_END(); const String *result = NULL; - if (value != NULL) + if (cfgOptionIdxSource(optionId, 0) != cfgSourceDefault) { - if (varType(value) == varTypeBool) - { - if (varBool(value)) - result = Y_STR; - else - result = N_STR; - } - else if (varType(value) == varTypeKeyValue) + const Variant *const value = cfgOptionIdxVar(optionId, optionIdx); + ASSERT(value != NULL); + + switch (varType(value)) { - String *resultTemp = strNew(); + case varTypeKeyValue: + { + String *resultTemp = strNew(); - const KeyValue *optionKv = varKv(value); - const VariantList *keyList = kvKeyList(optionKv); + const KeyValue *optionKv = varKv(value); + const VariantList *keyList = kvKeyList(optionKv); - for (unsigned int keyIdx = 0; keyIdx < varLstSize(keyList); keyIdx++) - { - if (keyIdx != 0) - strCatZ(resultTemp, ", "); + for (unsigned int keyIdx = 0; keyIdx < varLstSize(keyList); keyIdx++) + { + if (keyIdx != 0) + strCatZ(resultTemp, ", "); - strCatFmt( - resultTemp, "%s=%s", strZ(varStr(varLstGet(keyList, keyIdx))), - strZ(varStrForce(kvGet(optionKv, varLstGet(keyList, keyIdx))))); + strCatFmt( + resultTemp, "%s=%s", strZ(varStr(varLstGet(keyList, keyIdx))), + strZ(varStrForce(kvGet(optionKv, varLstGet(keyList, keyIdx))))); + } + + result = resultTemp; + break; } - result = resultTemp; - } - else if (varType(value) == varTypeVariantList) - { - String *resultTemp = strNew(); + case varTypeVariantList: + { + String *resultTemp = strNew(); - const VariantList *list = varVarLst(value); + const VariantList *list = varVarLst(value); - for (unsigned int listIdx = 0; listIdx < varLstSize(list); listIdx++) - { - if (listIdx != 0) - strCatZ(resultTemp, ", "); + for (unsigned int listIdx = 0; listIdx < varLstSize(list); listIdx++) + { + if (listIdx != 0) + strCatZ(resultTemp, ", "); - strCatFmt(resultTemp, "%s", strZ(varStr(varLstGet(list, listIdx)))); + strCatFmt(resultTemp, "%s", strZ(varStr(varLstGet(list, listIdx)))); + } + + result = resultTemp; + break; } - result = resultTemp; + default: + result = cfgOptionIdxDisplay(optionId, optionIdx); + break; } - else - result = cfgOptionDisplayVar(value, type); } FUNCTION_LOG_RETURN_CONST(STRING, result); @@ -386,7 +390,9 @@ if (strLstEmpty(cfgCommandParam())) { // Output command summary and description. Add a warning for internal commands. - CHECK(commandData[commandId].summary != NULL && commandData[commandId].description != NULL); + CHECK( + AssertError, commandData[commandId].summary != NULL && commandData[commandId].description != NULL, + "command help missing"); strCatFmt( result, @@ -450,10 +456,7 @@ // Output current and default values if they exist const String *defaultValue = cfgOptionDefault(optionId); - const String *value = NULL; - - if (cfgOptionIdxSource(optionId, 0) != cfgSourceDefault) - value = helpRenderValue(cfgOptionIdxVar(optionId, 0), cfgParseOptionType(optionId)); + const String *value = helpRenderValue(optionId, 0); if (value != NULL || defaultValue != NULL) { @@ -501,7 +504,9 @@ THROW_FMT(OptionInvalidError, "option '%s' is not valid for command '%s'", strZ(optionName), commandName); // Output option summary and description. Add a warning for internal options. - CHECK(optionData[option.id].summary != NULL && optionData[option.id].description != NULL); + CHECK( + AssertError, optionData[option.id].summary != NULL && optionData[option.id].description != NULL, + "option help missing"); strCatFmt( result, @@ -517,10 +522,7 @@ // Output current and default values if they exist const String *defaultValue = cfgOptionDefault(option.id); - const String *value = NULL; - - if (cfgOptionIdxSource(option.id, 0) != cfgSourceDefault) - value = helpRenderValue(cfgOptionIdxVar(option.id, 0), cfgParseOptionType(option.id)); + const String *value = helpRenderValue(option.id, 0); if (value != NULL || defaultValue != NULL) { diff -Nru pgbackrest-2.36/src/command/remote/remote.c pgbackrest-2.37/src/command/remote/remote.c --- pgbackrest-2.36/src/command/remote/remote.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/remote/remote.c 2022-01-03 13:43:55.000000000 +0000 @@ -41,7 +41,7 @@ TRY_BEGIN() { // Get the command. No need to check parameters since we know this is the first noop. - CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP); + CHECK(FormatError, protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP, "noop expected"); // Only try the lock if this is process 0, i.e. the remote started from the main process if (cfgOptionUInt(cfgOptProcess) == 0) diff -Nru pgbackrest-2.36/src/command/restore/restore.c pgbackrest-2.37/src/command/restore/restore.c --- pgbackrest-2.36/src/command/restore/restore.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/restore/restore.c 2022-01-03 13:43:55.000000000 +0000 @@ -167,10 +167,11 @@ } else { - LOG_WARN_FMT( - "automatic backup set selection cannot be performed with provided time '%s', latest backup set will be used" - "\nHINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone (+/- HH or HHMM or HH:MM)" - " - if timezone is omitted, local time is assumed (for UTC use +00)", + THROW_FMT( + FormatError, + "automatic backup set selection cannot be performed with provided time '%s'\n" + "HINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone (+/- HH or HHMM or HH:MM)" + " - if timezone is omitted, local time is assumed (for UTC use +00)", strZ(targetTime)); } } @@ -235,14 +236,11 @@ repoIdxMax = repoIdxMin; } - // Initialize a backup candidate list - List *backupCandidateList = lstNewP(sizeof(RestoreBackupData)); - + // If the set option was not provided by the user but a time to recover was set, then we will need to search for a backup + // set that satisfies the time condition, else we will use the backup provided const String *backupSetRequested = NULL; time_t timeTargetEpoch = 0; - // If the set option was not provided by the user but a time to recover was set, then we will need to search for a backup - // set that satisfies the time condition, else we will use the backup provided if (cfgOptionSource(cfgOptSet) == cfgSourceDefault) { if (cfgOptionStrId(cfgOptType) == CFGOPTVAL_TYPE_TIME) @@ -313,22 +311,13 @@ } } - // If a backup was found on this repo matching the criteria for time then exit, else determine if the latest - // backup from this repo might be used + // If a backup was found on this repo matching the criteria for time then exit if (found) break; - else - { - // If a backup was not yet found then set the latest from this repo as the backup that might be used - RestoreBackupData candidate = restoreBackupData( - latestBackup.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup))); - - lstAdd(backupCandidateList, &candidate); - } } + // Else use backup set found else { - // If the recovery type was not time (or time provided was not valid), then use the latest backup from this repo result = restoreBackupData(latestBackup.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup))); break; } @@ -356,17 +345,11 @@ { if (backupSetRequested != NULL) THROW_FMT(BackupSetInvalidError, "backup set %s is not valid", strZ(backupSetRequested)); - else if (timeTargetEpoch != 0 && lstSize(backupCandidateList) > 0) + else if (timeTargetEpoch != 0) { - // Since the repos were scanned in priority order, use the first candidate found - result = restoreBackupData( - ((RestoreBackupData *)lstGet(backupCandidateList, 0))->backupSet, - ((RestoreBackupData *)lstGet(backupCandidateList, 0))->repoIdx, - ((RestoreBackupData *)lstGet(backupCandidateList, 0))->backupCipherPass); - - LOG_WARN_FMT( - "unable to find backup set with stop time less than '%s', repo%u: latest backup set will be used", - strZ(cfgOptionDisplay(cfgOptTarget)), cfgOptionGroupIdxToKey(cfgOptGrpRepo, result.repoIdx)); + THROW_FMT( + BackupSetInvalidError, "unable to find backup set with stop time less than '%s'", + strZ(cfgOptionDisplay(cfgOptTarget))); } else THROW(BackupSetInvalidError, "no backup set found to restore"); @@ -394,7 +377,7 @@ MEM_CONTEXT_TEMP_BEGIN() { // If there are no files in the manifest then something has gone horribly wrong - CHECK(manifestFileTotal(manifest) > 0); + CHECK(FormatError, manifestFileTotal(manifest) > 0, "manifest missing files"); // Sanity check to ensure the manifest has not been moved to a new directory const ManifestData *data = manifestData(manifest); @@ -554,7 +537,7 @@ const ManifestPath *const path = manifestPathFindDefault(manifest, manifestName, NULL); const ManifestFile *const file = manifestFileFindDefault(manifest, manifestName, NULL); - CHECK(path == NULL || file == NULL); + CHECK(FormatError, path == NULL || file == NULL, "link may not be both file and path"); target = (ManifestTarget){.name = manifestName, .path = linkPath, .type = manifestTargetTypeLink}; @@ -590,7 +573,7 @@ target.path = linkPath; // The target must be a link since pg_data/ was prepended and pgdata is the only allowed path - CHECK(target.type == manifestTargetTypeLink); + CHECK(FormatError, target.type == manifestTargetTypeLink, "target must be a link"); // Error if the target is a tablespace if (target.tablespaceId != 0) @@ -940,7 +923,7 @@ { const ManifestFile *manifestFile = manifestFileFindDefault(cleanData->manifest, manifestName, NULL); - if (manifestFile != NULL) + if (manifestFile != NULL && manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL) { restoreCleanOwnership(pgPath, manifestFile->user, manifestFile->group, info->userId, info->groupId, false); restoreCleanMode(pgPath, manifestFile->mode, info); @@ -981,7 +964,7 @@ { const ManifestPath *manifestPath = manifestPathFindDefault(cleanData->manifest, manifestName, NULL); - if (manifestPath != NULL) + if (manifestPath != NULL && manifestLinkFindDefault(cleanData->manifest, manifestName, NULL) == NULL) { // Check ownership/permissions restoreCleanOwnership(pgPath, manifestPath->user, manifestPath->group, info->userId, info->groupId, false); @@ -1580,12 +1563,13 @@ kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_SUBPROCESS), NULL); kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_TIMESTAMP), NULL); kvPut(optionReplace, VARSTRDEF(CFGOPT_PROCESS_MAX), NULL); + kvPut(optionReplace, VARSTRDEF(CFGOPT_CMD), NULL); kvPut( result, VARSTRZ(RESTORE_COMMAND), VARSTR( strNewFmt( - "%s %s %%f \"%%p\"", strZ(cfgExe()), + "%s %s %%f \"%%p\"", strZ(cfgOptionStr(cfgOptCmd)), strZ(strLstJoin(cfgExecParam(cfgCmdArchiveGet, cfgCmdRoleMain, optionReplace, true, true), " "))))); } @@ -1606,7 +1590,7 @@ { // Write the recovery target kvPut( - result, VARSTR(strNewFmt(RECOVERY_TARGET "_%s", strZ(cfgOptionStr(cfgOptType)))), + result, VARSTR(strNewFmt(RECOVERY_TARGET "_%s", strZ(strIdToStr(cfgOptionStrId(cfgOptType))))), VARSTR(cfgOptionStr(cfgOptTarget))); // Write recovery_target_inclusive @@ -1948,8 +1932,8 @@ MEM_CONTEXT_TEMP_BEGIN() { - // Create list of process queue - *queueList = lstNewP(sizeof(List *)); + // Create list of process queues (use void * instead of List * to avoid Coverity false positive) + *queueList = lstNewP(sizeof(void *)); // Generate the list of processing queues (there is always at least one) StringList *targetList = strLstNew(); @@ -1985,7 +1969,7 @@ do { // A target should always be found - CHECK(targetIdx < strLstSize(targetList)); + CHECK(FormatError, targetIdx < strLstSize(targetList), "backup target not found"); if (strBeginsWith(file->name, strLstGet(targetList, targetIdx))) break; diff -Nru pgbackrest-2.36/src/command/server/ping.c pgbackrest-2.37/src/command/server/ping.c --- pgbackrest-2.36/src/command/server/ping.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/server/ping.c 2022-01-03 13:43:55.000000000 +0000 @@ -30,9 +30,8 @@ // Connect to server without any verification const TimeMSec timeout = cfgOptionUInt64(cfgOptIoTimeout); - IoClient *const tlsClient = tlsClientNew( - sckClientNew(host, cfgOptionUInt(cfgOptTlsServerPort), timeout, timeout), host, timeout, timeout, false, NULL, NULL, - NULL, NULL); + IoClient *const tlsClient = tlsClientNewP( + sckClientNew(host, cfgOptionUInt(cfgOptTlsServerPort), timeout, timeout), host, timeout, timeout, false); IoSession *const tlsSession = ioClientOpen(tlsClient); // Send ping diff -Nru pgbackrest-2.36/src/command/server/server.c pgbackrest-2.37/src/command/server/server.c --- pgbackrest-2.36/src/command/server/server.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/server/server.c 2022-01-03 13:43:55.000000000 +0000 @@ -8,74 +8,206 @@ #include "command/remote/remote.h" #include "command/server/server.h" #include "common/debug.h" +#include "common/exit.h" #include "common/fork.h" #include "common/io/socket/server.h" #include "common/io/tls/server.h" #include "config/config.h" +#include "config/load.h" #include "protocol/helper.h" +/*********************************************************************************************************************************** +Local variables +***********************************************************************************************************************************/ +static struct ServerLocal +{ + MemContext *memContext; // Mem context for server + + unsigned int argListSize; // Argument list size + const char **argList; // Argument list + + List *processList; // List of child processes + + bool sigHup; // SIGHUP was caught + bool sigTerm; // SIGTERM was caught + + IoServer *socketServer; // Socket server + IoServer *tlsServer; // TLS server +} serverLocal; + +/*********************************************************************************************************************************** +Initialization can be redone when options change +***********************************************************************************************************************************/ +static void +cmdServerInit(void) +{ + // Initialize mem context + if (serverLocal.memContext == NULL) + { + MEM_CONTEXT_BEGIN(memContextTop()) + { + MEM_CONTEXT_NEW_BEGIN("Server") + { + serverLocal.memContext = MEM_CONTEXT_NEW(); + serverLocal.processList = lstNewP(sizeof(pid_t)); + } + MEM_CONTEXT_NEW_END(); + } + MEM_CONTEXT_END(); + } + + MEM_CONTEXT_BEGIN(serverLocal.memContext) + { + // Free old servers + ioServerFree(serverLocal.socketServer); + ioServerFree(serverLocal.tlsServer); + + // Create new servers + serverLocal.socketServer = sckServerNew( + cfgOptionStr(cfgOptTlsServerAddress), cfgOptionUInt(cfgOptTlsServerPort), cfgOptionUInt64(cfgOptProtocolTimeout)); + serverLocal.tlsServer = tlsServerNew( + cfgOptionStr(cfgOptTlsServerAddress), cfgOptionStr(cfgOptTlsServerCaFile), cfgOptionStr(cfgOptTlsServerKeyFile), + cfgOptionStr(cfgOptTlsServerCertFile), cfgOptionUInt64(cfgOptProtocolTimeout)); + } + MEM_CONTEXT_END(); +} + +/*********************************************************************************************************************************** +Handlers to set flags on signals +***********************************************************************************************************************************/ +static void +cmdServerSigHup(const int signalType) +{ + (void)signalType; + serverLocal.sigHup = true; +} + +static void +cmdServerSigTerm(const int signalType) +{ + (void)signalType; + serverLocal.sigTerm = true; +} + +/*********************************************************************************************************************************** +Handler to reap child processes +***********************************************************************************************************************************/ +static void +cmdServerSigChild(const int signalType, siginfo_t *signalInfo, void *context) +{ + (void)signalType; + (void)context; + + ASSERT(signalInfo->si_code == CLD_EXITED); + + // Find the process and remove it + for (unsigned int processIdx = 0; processIdx < lstSize(serverLocal.processList); processIdx++) + { + if (*(int *)lstGet(serverLocal.processList, processIdx) == signalInfo->si_pid) + lstRemoveIdx(serverLocal.processList, processIdx); + } +} + /**********************************************************************************************************************************/ void -cmdServer(uint64_t connectionMax) +cmdServer(const unsigned int argListSize, const char *argList[]) { - FUNCTION_LOG_VOID(logLevelDebug); - - ASSERT(connectionMax > 0); + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(UINT, argListSize); + FUNCTION_LOG_PARAM(CHARPY, argList); + FUNCTION_LOG_END(); + + // Initialize server + cmdServerInit(); + + // Set arguments used for reload + serverLocal.argListSize = argListSize; + serverLocal.argList = argList; MEM_CONTEXT_TEMP_BEGIN() { - IoServer *const tlsServer = tlsServerNew( - cfgOptionStr(cfgOptTlsServerAddress), cfgOptionStr(cfgOptTlsServerCaFile), cfgOptionStr(cfgOptTlsServerKeyFile), - cfgOptionStr(cfgOptTlsServerCertFile), cfgOptionUInt64(cfgOptProtocolTimeout)); - IoServer *const socketServer = sckServerNew( - cfgOptionStr(cfgOptTlsServerAddress), cfgOptionUInt(cfgOptTlsServerPort), cfgOptionUInt64(cfgOptProtocolTimeout)); + // Set signal handlers + sigaction(SIGHUP, &(struct sigaction){.sa_handler = cmdServerSigHup}, NULL); + sigaction(SIGTERM, &(struct sigaction){.sa_handler = cmdServerSigTerm}, NULL); + sigaction( + SIGCHLD, &(struct sigaction){.sa_sigaction = cmdServerSigChild, .sa_flags = SA_NOCLDSTOP | SA_NOCLDWAIT | SA_SIGINFO}, + NULL); - // Accept connections until connection max is reached + // Accept connections indefinitely. The only way to exit this loop is for the process to receive a signal. do { // Accept a new connection - IoSession *const socketSession = ioServerAccept(socketServer, NULL); + IoSession *const socketSession = ioServerAccept(serverLocal.socketServer, NULL); - // Fork off the child process - pid_t pid = forkSafe(); - - if (pid == 0) + if (socketSession != NULL) { - // Close the server socket so we don't hold the port open if the parent exits first - ioServerFree(socketServer); + // Fork off the child process + pid_t pid = forkSafe(); + + if (pid == 0) + { + // Reset SIGCHLD to default + sigaction(SIGCHLD, &(struct sigaction){.sa_handler = SIG_DFL}, NULL); + + // Set standard signal handlers + exitInit(); - // Disable logging and close log file - logClose(); + // Close the server socket so we don't hold the port open if the parent exits first + ioServerFree(serverLocal.socketServer); - // Detach from parent process - forkDetach(); + // Disable logging and close log file + logClose(); - // Start standard remote processing if a server is returned - ProtocolServer *server = protocolServer(tlsServer, socketSession); + // Start standard remote processing if a server is returned + ProtocolServer *server = protocolServer(serverLocal.tlsServer, socketSession); - if (server != NULL) - cmdRemote(server); + if (server != NULL) + cmdRemote(server); - break; + break; + } + // Add process to list + else + lstAdd(serverLocal.processList, &pid); + + // Free the socket since the child is now using it + ioSessionFree(socketSession); } - // Wait for first fork to exit - else + + // Reload configuration + if (serverLocal.sigHup) { - // The process that was just forked should return immediately - int processStatus; + LOG_DETAIL("configuration reload begin"); - THROW_ON_SYS_ERROR(waitpid(pid, &processStatus, 0) == -1, ExecuteError, "unable to wait for forked process"); + // Reload configuration + cfgLoad(serverLocal.argListSize, serverLocal.argList); - // The first fork should exit with success. If not, something went wrong during the second fork. - CHECK(WIFEXITED(processStatus) && WEXITSTATUS(processStatus) == 0); - } + // Reinitialize server + cmdServerInit(); - // Free the socket since the child is now using it - ioSessionFree(socketSession); + LOG_DETAIL("configuration reload end"); + + // Reset flag + serverLocal.sigHup = false; + } } - while (--connectionMax > 0); + while (!serverLocal.sigTerm); } MEM_CONTEXT_TEMP_END(); + // Terminate any remaining children on SIGTERM. Disable the callback so it does not fire in the middle of the loop. + if (serverLocal.sigTerm) + { + sigaction(SIGCHLD, &(struct sigaction){.sa_flags = SA_NOCLDSTOP | SA_NOCLDWAIT}, NULL); + + for (unsigned int processIdx = 0; processIdx < lstSize(serverLocal.processList); processIdx++) + { + pid_t pid = *(int *)lstGet(serverLocal.processList, processIdx); + + LOG_WARN_FMT("terminate child process %d", pid); + kill(pid, SIGTERM); + } + } + FUNCTION_LOG_RETURN_VOID(); } diff -Nru pgbackrest-2.36/src/command/server/server.h pgbackrest-2.37/src/command/server/server.h --- pgbackrest-2.36/src/command/server/server.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/server/server.h 2022-01-03 13:43:55.000000000 +0000 @@ -9,6 +9,6 @@ /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -void cmdServer(uint64_t connectionMax); +void cmdServer(unsigned int argListSize, const char *argList[]); #endif diff -Nru pgbackrest-2.36/src/command/stanza/common.c pgbackrest-2.37/src/command/stanza/common.c --- pgbackrest-2.36/src/command/stanza/common.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/stanza/common.c 2022-01-03 13:43:55.000000000 +0000 @@ -50,10 +50,10 @@ DbGetResult dbObject = dbGet(false, true, false); // Get the pgControl information from the pg*-path deemed to be the primary - result = pgControlFromFile(storagePgIdx(dbObject.primaryIdx)); + result = dbPgControl(dbObject.primary); // Check the user configured path and version against the database - checkDbConfig(result.version, dbObject.primaryIdx, dbObject.primary, false); + checkDbConfig(dbPgControl(dbObject.primary).version, dbObject.primaryIdx, dbObject.primary, false); } // If the database is not online, assume that pg1 is the primary else diff -Nru pgbackrest-2.36/src/command/stanza/delete.c pgbackrest-2.37/src/command/stanza/delete.c --- pgbackrest-2.36/src/command/stanza/delete.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/stanza/delete.c 2022-01-03 13:43:55.000000000 +0000 @@ -17,39 +17,6 @@ #include "protocol/helper.h" #include "storage/helper.h" -/*********************************************************************************************************************************** -Helper functions to assist with testing -***********************************************************************************************************************************/ -static void -manifestDelete(const Storage *storageRepoWriteStanza) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STORAGE, storageRepoWriteStanza); - FUNCTION_TEST_END(); - - ASSERT(storageRepoWriteStanza != NULL); - - // Get the list of backup directories from newest to oldest since don't want to invalidate a backup before - // invalidating any backups that depend on it. - StringList *backupList = strLstSort( - storageListP( - storageRepo(), STORAGE_REPO_BACKUP_STR, - .expression = backupRegExpP(.full = true, .differential = true, .incremental = true)), - sortOrderDesc); - - // Delete all manifest files - for (unsigned int idx = 0; idx < strLstSize(backupList); idx++) - { - storageRemoveP( - storageRepoWriteStanza, strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(strLstGet(backupList, idx)))); - storageRemoveP( - storageRepoWriteStanza, - strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(strLstGet(backupList, idx)))); - } - - FUNCTION_TEST_RETURN_VOID(); -} - static bool stanzaDelete(const Storage *storageRepoWriteStanza, const StringList *archiveList, const StringList *backupList) { @@ -106,9 +73,6 @@ storageRemoveP(storageRepoWriteStanza, INFO_BACKUP_PATH_FILE_STR); storageRemoveP(storageRepoWriteStanza, INFO_BACKUP_PATH_FILE_COPY_STR); } - - // Remove manifest files - manifestDelete(storageRepoWriteStanza); } // Recursively remove the entire stanza repo if exists. S3 will attempt to remove even if not. diff -Nru pgbackrest-2.36/src/command/verify/verify.c pgbackrest-2.37/src/command/verify/verify.c --- pgbackrest-2.36/src/command/verify/verify.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/command/verify/verify.c 2022-01-03 13:43:55.000000000 +0000 @@ -1152,7 +1152,7 @@ // Get the last backup as current if it is not in backup.info current list String *backupLabel = strLstGet(backupList, strLstSize(backupList) - 1); - if (infoBackupDataByLabel(backupInfo, backupLabel) == NULL) + if (!infoBackupLabelExists(backupInfo, backupLabel)) { // Duplicate the string into the prior context MEM_CONTEXT_PRIOR_BEGIN() diff -Nru pgbackrest-2.36/src/common/assert.h pgbackrest-2.37/src/common/assert.h --- pgbackrest-2.36/src/common/assert.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/assert.h 2022-01-03 13:43:55.000000000 +0000 @@ -44,11 +44,11 @@ /*********************************************************************************************************************************** Checks are used in production builds to test very important conditions. Be sure to limit use to the most critical cases. ***********************************************************************************************************************************/ -#define CHECK(condition) \ +#define CHECK(type, condition, message) \ do \ { \ if (!(condition)) \ - THROW_FMT(AssertError, "check '%s' failed", #condition); \ + THROW(type, message); \ } \ while (0) diff -Nru pgbackrest-2.36/src/common/compress/helper.c pgbackrest-2.37/src/common/compress/helper.c --- pgbackrest-2.36/src/common/compress/helper.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/compress/helper.c 2022-01-03 13:43:55.000000000 +0000 @@ -35,6 +35,7 @@ ***********************************************************************************************************************************/ static const struct CompressHelperLocal { + StringId typeId; // Compress type id const String *const type; // Compress type -- must be extension without period prefixed const String *const ext; // File extension with period prefixed StringId compressType; // Type of the compression filter @@ -45,10 +46,12 @@ } compressHelperLocal[] = { { + .typeId = STRID5("none", 0x2b9ee0), .type = STRDEF(COMPRESS_TYPE_NONE), .ext = STRDEF(""), }, { + .typeId = STRID5("bz2", 0x73420), .type = STRDEF(BZ2_EXT), .ext = STRDEF("." BZ2_EXT), .compressType = BZ2_COMPRESS_FILTER_TYPE, @@ -58,6 +61,7 @@ .levelDefault = 9, }, { + .typeId = STRID5("gz", 0x3470), .type = STRDEF(GZ_EXT), .ext = STRDEF("." GZ_EXT), .compressType = GZ_COMPRESS_FILTER_TYPE, @@ -67,6 +71,7 @@ .levelDefault = 6, }, { + .typeId = STRID6("lz4", 0x2068c1), .type = STRDEF(LZ4_EXT), .ext = STRDEF("." LZ4_EXT), #ifdef HAVE_LIBLZ4 @@ -78,6 +83,7 @@ #endif }, { + .typeId = STRID5("zst", 0x527a0), .type = STRDEF(ZST_EXT), .ext = STRDEF("." ZST_EXT), #ifdef HAVE_LIBZST @@ -89,13 +95,10 @@ #endif }, { + .typeId = STRID5("xz", 0x3580), .type = STRDEF(XZ_EXT), .ext = STRDEF("." XZ_EXT), }, - { - .type = STRDEF(BZ2_EXT), - .ext = STRDEF("." BZ2_EXT), - }, }; #define COMPRESS_LIST_SIZE \ @@ -103,24 +106,24 @@ /**********************************************************************************************************************************/ CompressType -compressTypeEnum(const String *type) +compressTypeEnum(const StringId type) { FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, type); + FUNCTION_TEST_PARAM(STRING_ID, type); FUNCTION_TEST_END(); - ASSERT(type != NULL); + ASSERT(type != 0); CompressType result = compressTypeNone; for (; result < COMPRESS_LIST_SIZE; result++) { - if (strEq(type, compressHelperLocal[result].type)) + if (type == compressHelperLocal[result].typeId) break; } if (result == COMPRESS_LIST_SIZE) - THROW_FMT(AssertError, "invalid compression type '%s'", strZ(type)); + THROW_FMT(AssertError, "invalid compression type '%s'", strZ(strIdToStr(type))); FUNCTION_TEST_RETURN(result); } diff -Nru pgbackrest-2.36/src/common/compress/helper.h pgbackrest-2.37/src/common/compress/helper.h --- pgbackrest-2.36/src/common/compress/helper.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/compress/helper.h 2022-01-03 13:43:55.000000000 +0000 @@ -26,7 +26,7 @@ compressTypeXz, // xz/lzma } CompressType; -#include +#include #include /*********************************************************************************************************************************** @@ -40,7 +40,7 @@ Functions ***********************************************************************************************************************************/ // Get enum from a compression type string -CompressType compressTypeEnum(const String *type); +CompressType compressTypeEnum(StringId type); // Check that a valid compress type is compiled into this binary. Errors when the compress type is not present. void compressTypePresent(CompressType type); diff -Nru pgbackrest-2.36/src/common/error.c pgbackrest-2.37/src/common/error.c --- pgbackrest-2.36/src/common/error.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/error.c 2022-01-03 13:43:55.000000000 +0000 @@ -42,7 +42,7 @@ /*********************************************************************************************************************************** States for each try ***********************************************************************************************************************************/ -typedef enum {errorStateBegin, errorStateTry, errorStateCatch, errorStateFinal, errorStateEnd} ErrorState; +typedef enum {errorStateTry, errorStateCatch, errorStateEnd} ErrorState; /*********************************************************************************************************************************** Track error handling @@ -260,27 +260,19 @@ } /**********************************************************************************************************************************/ -bool -errorInternalStateTry(void) -{ - return errorInternalState() == errorStateTry; -} - -/**********************************************************************************************************************************/ -bool -errorInternalStateCatch(const ErrorType *errorTypeCatch) +void +errorInternalTryBegin(const char *const fileName, const char *const functionName, const int fileLine) { - if (errorInternalState() == errorStateCatch && errorInstanceOf(errorTypeCatch)) - return errorInternalProcess(true); + // If try total has been exceeded then throw an error + if (errorContext.tryTotal >= ERROR_TRY_MAX) + errorInternalThrowFmt(&AssertError, fileName, functionName, fileLine, "too many nested try blocks"); - return false; -} + // Increment try total + errorContext.tryTotal++; -/**********************************************************************************************************************************/ -bool -errorInternalStateFinal(void) -{ - return errorInternalState() == errorStateFinal; + // Setup try + errorContext.tryList[errorContext.tryTotal].state = errorStateTry; + errorContext.tryList[errorContext.tryTotal].uncaught = false; } /**********************************************************************************************************************************/ @@ -292,21 +284,26 @@ /**********************************************************************************************************************************/ bool -errorInternalTry(const char *fileName, const char *functionName, int fileLine) +errorInternalCatch(const ErrorType *const errorTypeCatch) { - // If try total has been exceeded then throw an error - if (errorContext.tryTotal >= ERROR_TRY_MAX) - errorInternalThrowFmt(&AssertError, fileName, functionName, fileLine, "too many nested try blocks"); + // If just entering error state clean up the stack + if (errorInternalState() == errorStateTry) + { + for (unsigned int handlerIdx = 0; handlerIdx < errorContext.handlerTotal; handlerIdx++) + errorContext.handlerList[handlerIdx](errorTryDepth()); - // Increment try total - errorContext.tryTotal++; + errorContext.tryList[errorContext.tryTotal].state++; + } - // Setup try - errorContext.tryList[errorContext.tryTotal].state = errorStateBegin; - errorContext.tryList[errorContext.tryTotal].uncaught = false; + if (errorInternalState() == errorStateCatch && errorInstanceOf(errorTypeCatch)) + { + errorContext.tryList[errorContext.tryTotal].uncaught = false; + errorContext.tryList[errorContext.tryTotal].state++; + + return true; + } - // Try setup was successful - return true; + return false; } /**********************************************************************************************************************************/ @@ -331,52 +328,22 @@ } /**********************************************************************************************************************************/ -bool -errorInternalProcess(bool catch) +void +errorInternalTryEnd(void) { - // If a catch statement then return - if (catch) - { - errorContext.tryList[errorContext.tryTotal].uncaught = false; - return true; - } - // Else if just entering error state clean up the stack - else if (errorContext.tryList[errorContext.tryTotal].state == errorStateTry) - { - for (unsigned int handlerIdx = 0; handlerIdx < errorContext.handlerTotal; handlerIdx++) - errorContext.handlerList[handlerIdx](errorTryDepth()); - } - // Any catch blocks have been processed and none of them called RETHROW() so clear the error - if (errorContext.tryList[errorContext.tryTotal].state == errorStateCatch && + if (errorContext.tryList[errorContext.tryTotal].state == errorStateEnd && !errorContext.tryList[errorContext.tryTotal].uncaught) { errorContext.error = (Error){0}; } - // Increment the state - errorContext.tryList[errorContext.tryTotal].state++; - - // If the error has been caught then increment the state - if (errorContext.tryList[errorContext.tryTotal].state == errorStateCatch && - !errorContext.tryList[errorContext.tryTotal].uncaught) - { - errorContext.tryList[errorContext.tryTotal].state++; - } - - // Return if not done - if (errorContext.tryList[errorContext.tryTotal].state < errorStateEnd) - return true; - // Remove the try errorContext.tryTotal--; // If not caught in the last try then propagate if (errorContext.tryList[errorContext.tryTotal + 1].uncaught) errorInternalPropagate(); - - // Nothing left to process - return false; } /**********************************************************************************************************************************/ diff -Nru pgbackrest-2.36/src/common/error.h pgbackrest-2.37/src/common/error.h --- pgbackrest-2.36/src/common/error.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/error.h 2022-01-03 13:43:55.000000000 +0000 @@ -126,37 +126,43 @@ #define TRY_BEGIN() \ do \ { \ - if (errorInternalTry(__FILE__, __func__, __LINE__) && setjmp(*errorInternalJump()) >= 0) \ - { \ - while (errorInternalProcess(false)) \ - { \ - if (errorInternalStateTry()) + errorInternalTryBegin(__FILE__, __func__, __LINE__); \ + \ + if (setjmp(*errorInternalJump()) == 0) \ + { /*********************************************************************************************************************************** Catch a specific error thrown in the try block ***********************************************************************************************************************************/ #define CATCH(errorTypeCatch) \ - else if (errorInternalStateCatch(&errorTypeCatch)) + } \ + else if (errorInternalCatch(&errorTypeCatch)) \ + { /*********************************************************************************************************************************** Catch any error thrown in the try block ***********************************************************************************************************************************/ #define CATCH_ANY() \ - else if (errorInternalStateCatch(&RuntimeError)) + } \ + else if (errorInternalCatch(&RuntimeError)) \ + { /*********************************************************************************************************************************** Code to run whether the try block was successful or not ***********************************************************************************************************************************/ #define FINALLY() \ - else if (errorInternalStateFinal()) + } \ + { /*********************************************************************************************************************************** End the try block ***********************************************************************************************************************************/ #define TRY_END() \ - } \ } \ - } while (0) + \ + errorInternalTryEnd(); \ + } \ + while (0) /*********************************************************************************************************************************** Throw an error @@ -279,26 +285,20 @@ These functions are used by the macros to implement the error handler and should never be called independently. ***********************************************************************************************************************************/ // Begin the try block -bool errorInternalTry(const char *fileName, const char *functionName, int fileLine); +void errorInternalTryBegin(const char *fileName, const char *functionName, int fileLine); // Return jump buffer for current try jmp_buf *errorInternalJump(void); -// True when in try state -bool errorInternalStateTry(void); - // True when in catch state and the expected error matches -bool errorInternalStateCatch(const ErrorType *errorTypeCatch); - -// True when in final state -bool errorInternalStateFinal(void); - -// Process the error through each try and state -bool errorInternalProcess(bool catch); +bool errorInternalCatch(const ErrorType *errorTypeCatch); // Propagate the error up so it can be caught void errorInternalPropagate(void) __attribute__((__noreturn__)); +// End the try block +void errorInternalTryEnd(void); + // Throw an error void errorInternalThrow( const ErrorType *errorType, const char *fileName, const char *functionName, int fileLine, const char *message, diff -Nru pgbackrest-2.36/src/common/io/client.h pgbackrest-2.37/src/common/io/client.h --- pgbackrest-2.36/src/common/io/client.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/client.h 2022-01-03 13:43:55.000000000 +0000 @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Io Client Interface -Create sessions for protocol clients. For example, a TLS client can be created with tlsClientNew() and then new TLS sessions can be +Create sessions for protocol clients. For example, a TLS client can be created with tlsClientNewP() and then new TLS sessions can be opened with ioClientOpen(). ***********************************************************************************************************************************/ #ifndef COMMON_IO_CLIENT_H diff -Nru pgbackrest-2.36/src/common/io/filter/filter.c pgbackrest-2.37/src/common/io/filter/filter.c --- pgbackrest-2.36/src/common/io/filter/filter.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/filter/filter.c 2022-01-03 13:43:55.000000000 +0000 @@ -67,8 +67,8 @@ ASSERT(this != NULL); ASSERT(this->pub.interface.in != NULL); - CHECK(input == NULL || !bufEmpty(input)); - CHECK(!this->flushing || input == NULL); + CHECK(AssertError, input == NULL || !bufEmpty(input), "expected data or flush"); + CHECK(AssertError, !this->flushing || input == NULL, "no data allowed after flush"); if (input == NULL) this->flushing = true; @@ -91,8 +91,8 @@ ASSERT(this != NULL); ASSERT(output != NULL); ASSERT(this->pub.interface.inOut != NULL); - CHECK(input == NULL || !bufEmpty(input)); - CHECK(!this->flushing || input == NULL); + CHECK(AssertError, input == NULL || !bufEmpty(input), "expected data or flush"); + CHECK(AssertError, !this->flushing || input == NULL, "no data allowed after flush"); if (input == NULL && !this->flushing) this->flushing = true; @@ -100,7 +100,8 @@ if (!ioFilterDone(this)) this->pub.interface.inOut(this->pub.driver, input, output); - CHECK(!ioFilterInputSame(this) || !bufEmpty(output)); + CHECK(AssertError, !ioFilterInputSame(this) || !bufEmpty(output), "expected input same or output"); + FUNCTION_TEST_RETURN_VOID(); } diff -Nru pgbackrest-2.36/src/common/io/http/request.c pgbackrest-2.37/src/common/io/http/request.c --- pgbackrest-2.36/src/common/io/http/request.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/http/request.c 2022-01-03 13:43:55.000000000 +0000 @@ -252,7 +252,7 @@ // Output path/query strCatZ(error, ":\n*** Path/Query ***:"); - strCatFmt(error, "\n%s", strZ(httpRequestPath(this))); + strCatFmt(error, "\n%s %s", strZ(httpRequestVerb(this)), strZ(httpRequestPath(this))); if (httpRequestQuery(this) != NULL) strCatFmt(error, "?%s", strZ(httpQueryRenderP(httpRequestQuery(this), .redact = true))); diff -Nru pgbackrest-2.36/src/common/io/socket/client.c pgbackrest-2.37/src/common/io/socket/client.c --- pgbackrest-2.36/src/common/io/socket/client.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/socket/client.c 2022-01-03 13:43:55.000000000 +0000 @@ -78,7 +78,7 @@ { // Assume there will be no retry retry = false; - int fd = -1; + volatile int fd = -1; TRY_BEGIN() { diff -Nru pgbackrest-2.36/src/common/io/socket/server.c pgbackrest-2.37/src/common/io/socket/server.c --- pgbackrest-2.36/src/common/io/socket/server.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/socket/server.c 2022-01-03 13:43:55.000000000 +0000 @@ -96,18 +96,21 @@ int serverSocket = accept(this->socket, (struct sockaddr *)&addr, &len); - THROW_ON_SYS_ERROR(serverSocket == -1, FileOpenError, "unable to accept socket"); + if (serverSocket != -1) + { + // Create socket session + sckOptionSet(serverSocket); - // Create socket session - sckOptionSet(serverSocket); + MEM_CONTEXT_PRIOR_BEGIN() + { + result = sckSessionNew(ioSessionRoleServer, serverSocket, this->address, this->port, this->timeout); + } + MEM_CONTEXT_PRIOR_END(); - MEM_CONTEXT_PRIOR_BEGIN() - { - result = sckSessionNew(ioSessionRoleServer, serverSocket, this->address, this->port, this->timeout); + statInc(SOCKET_STAT_SESSION_STR); } - MEM_CONTEXT_PRIOR_END(); - - statInc(SOCKET_STAT_SESSION_STR); + else + THROW_ON_SYS_ERROR(errno != EINTR, FileOpenError, "unable to accept socket"); } MEM_CONTEXT_TEMP_END(); diff -Nru pgbackrest-2.36/src/common/io/tls/client.c pgbackrest-2.37/src/common/io/tls/client.c --- pgbackrest-2.36/src/common/io/tls/client.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/tls/client.c 2022-01-03 13:43:55.000000000 +0000 @@ -348,8 +348,7 @@ IoClient * tlsClientNew( IoClient *const ioClient, const String *const host, const TimeMSec timeoutConnect, const TimeMSec timeoutSession, - const bool verifyPeer, const String *const caFile, const String *const caPath, const String *const certFile, - const String *const keyFile) + const bool verifyPeer, const TlsClientNewParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(IO_CLIENT, ioClient); @@ -357,10 +356,10 @@ FUNCTION_LOG_PARAM(TIME_MSEC, timeoutConnect); FUNCTION_LOG_PARAM(TIME_MSEC, timeoutSession); FUNCTION_LOG_PARAM(BOOL, verifyPeer); - FUNCTION_LOG_PARAM(STRING, caFile); - FUNCTION_LOG_PARAM(STRING, caPath); - FUNCTION_LOG_PARAM(STRING, certFile); - FUNCTION_LOG_PARAM(STRING, keyFile); + FUNCTION_LOG_PARAM(STRING, param.caFile); + FUNCTION_LOG_PARAM(STRING, param.caPath); + FUNCTION_LOG_PARAM(STRING, param.certFile); + FUNCTION_LOG_PARAM(STRING, param.keyFile); FUNCTION_LOG_END(); ASSERT(ioClient != NULL); @@ -391,10 +390,11 @@ if (driver->verifyPeer) { // If the user specified a location - if (caFile != NULL || caPath != NULL) // {vm_covered} + if (param.caFile != NULL || param.caPath != NULL) // {vm_covered} { cryptoError( // {vm_covered} - SSL_CTX_load_verify_locations(driver->context, strZNull(caFile), strZNull(caPath)) != 1, // {vm_covered} + SSL_CTX_load_verify_locations( // {vm_covered} + driver->context, strZNull(param.caFile), strZNull(param.caPath)) != 1, // {vm_covered} "unable to set user-defined CA certificate location"); // {vm_covered} } // Else use the defaults @@ -406,7 +406,7 @@ } // Load certificate and key, if specified - tlsCertKeyLoad(driver->context, certFile, keyFile); + tlsCertKeyLoad(driver->context, param.certFile, param.keyFile); // Increment stat statInc(TLS_STAT_CLIENT_STR); diff -Nru pgbackrest-2.36/src/common/io/tls/client.h pgbackrest-2.37/src/common/io/tls/client.h --- pgbackrest-2.36/src/common/io/tls/client.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/tls/client.h 2022-01-03 13:43:55.000000000 +0000 @@ -30,9 +30,21 @@ /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ +typedef struct TlsClientNewParam +{ + VAR_PARAM_HEADER; + const String *caFile; + const String *caPath; + const String *certFile; + const String *keyFile; +} TlsClientNewParam; + +#define tlsClientNewP(ioClient, host, timeoutConnect, timeoutSession, verifyPeer, ...) \ + tlsClientNew(ioClient, host, timeoutConnect, timeoutSession, verifyPeer, (TlsClientNewParam){VAR_PARAM_INIT, __VA_ARGS__}) + IoClient *tlsClientNew( - IoClient *ioClient, const String *host, TimeMSec timeoutConnect, TimeMSec timeoutSession, bool verifyPeer, const String *caFile, - const String *caPath, const String *certFile, const String *keyFile); + IoClient *ioClient, const String *host, TimeMSec timeoutConnect, TimeMSec timeoutSession, bool verifyPeer, + TlsClientNewParam param); /*********************************************************************************************************************************** Functions diff -Nru pgbackrest-2.36/src/common/io/tls/common.c pgbackrest-2.37/src/common/io/tls/common.c --- pgbackrest-2.36/src/common/io/tls/common.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/tls/common.c 2022-01-03 13:43:55.000000000 +0000 @@ -60,10 +60,10 @@ FUNCTION_TEST_END(); // {vm_covered} X509_NAME *const subjectName = X509_get_subject_name(certificate); // {vm_covered} - CHECK(subjectName != NULL); // {vm_covered} + CHECK(FormatError, subjectName != NULL, "subject name is missing"); // {vm_covered} const int commonNameIndex = X509_NAME_get_index_by_NID(subjectName, NID_commonName, -1); // {vm_covered} - CHECK(commonNameIndex >= 0); // {vm_covered} + CHECK(FormatError, commonNameIndex >= 0, "common name is missing"); // {vm_covered} String *result = tlsAsn1ToStr(X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subjectName, commonNameIndex))); // {vm_covered} @@ -78,7 +78,7 @@ static int tlsCertPwd(char *buffer, const int size, const int rwFlag, void *const userData) { - CHECK(size > 0); + CHECK(ServiceError, size > 0, "buffer has zero size"); (void)rwFlag; (void)userData; // No password is currently supplied diff -Nru pgbackrest-2.36/src/common/io/tls/session.c pgbackrest-2.37/src/common/io/tls/session.c --- pgbackrest-2.36/src/common/io/tls/session.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/io/tls/session.c 2022-01-03 13:43:55.000000000 +0000 @@ -267,8 +267,7 @@ result = tlsSessionResult(this, SSL_write(this->session, bufPtrConst(buffer), (int)bufUsed(buffer)), false); - // Either a retry or all data was written - CHECK(result == 0 || (size_t)result == bufUsed(buffer)); + CHECK(ServiceError, result == 0 || (size_t)result == bufUsed(buffer), "expected retry or complete write"); } FUNCTION_LOG_RETURN_VOID(); diff -Nru pgbackrest-2.36/src/common/log.c pgbackrest-2.37/src/common/log.c --- pgbackrest-2.36/src/common/log.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/log.c 2022-01-03 13:43:55.000000000 +0000 @@ -65,38 +65,64 @@ /**********************************************************************************************************************************/ #define LOG_LEVEL_TOTAL (LOG_LEVEL_MAX + 1) -static const char *const logLevelList[LOG_LEVEL_TOTAL] = +static const struct LogLevel { - "OFF", - "ASSERT", - "ERROR", - "PROTOCOL", - "WARN", - "INFO", - "DETAIL", - "DEBUG", - "TRACE", + const StringId id; // Id + const char *const name; // Name +} logLevelList[LOG_LEVEL_TOTAL] = +{ + { + .id = STRID5("off", 0x18cf0), + .name = "OFF", + }, + { + // No id here because this level is not user selectable + .name = "ASSERT", + }, + { + .id = STRID5("error", 0x127ca450), + .name = "ERROR", + }, + { + .id = STRID5("warn", 0x748370), + .name = "WARN", + }, + { + .id = STRID5("info", 0x799c90), + .name = "INFO", + }, + { + .id = STRID5("detail", 0x1890d0a40), + .name = "DETAIL", + }, + { + .id = STRID5("debug", 0x7a88a40), + .name = "DEBUG", + }, + { + .id = STRID5("trace", 0x5186540), + .name = "TRACE", + }, }; LogLevel -logLevelEnum(const char *logLevel) +logLevelEnum(const StringId logLevelId) { FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRINGZ, logLevel); + FUNCTION_TEST_PARAM(STRING_ID, logLevelId); FUNCTION_TEST_END(); - ASSERT(logLevel != NULL); + ASSERT(logLevelId != 0); LogLevel result = logLevelOff; // Search for the log level for (; result < LOG_LEVEL_TOTAL; result++) - if (strcasecmp(logLevel, logLevelList[result]) == 0) + if (logLevelId == logLevelList[result].id) break; - // If the log level was not found - if (result == LOG_LEVEL_TOTAL) - THROW_FMT(AssertError, "log level '%s' not found", logLevel); + // Check that the log level was found + CHECK(AssertError, result != LOG_LEVEL_TOTAL, "invalid log level"); FUNCTION_TEST_RETURN(result); } @@ -110,7 +136,7 @@ ASSERT(logLevel <= LOG_LEVEL_MAX); - FUNCTION_TEST_RETURN(logLevelList[logLevel]); + FUNCTION_TEST_RETURN(logLevelList[logLevel].name); } /**********************************************************************************************************************************/ diff -Nru pgbackrest-2.36/src/common/log.h pgbackrest-2.37/src/common/log.h --- pgbackrest-2.36/src/common/log.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/log.h 2022-01-03 13:43:55.000000000 +0000 @@ -8,6 +8,7 @@ #include #include "common/logLevel.h" +#include "common/type/stringId.h" /*********************************************************************************************************************************** Max size allowed for a single log message including header @@ -36,7 +37,7 @@ bool logAny(LogLevel logLevel); // Convert log level to string and vice versa -LogLevel logLevelEnum(const char *logLevel); +LogLevel logLevelEnum(StringId logLevelId); const char *logLevelStr(LogLevel logLevel); /*********************************************************************************************************************************** diff -Nru pgbackrest-2.36/src/common/logLevel.h pgbackrest-2.37/src/common/logLevel.h --- pgbackrest-2.36/src/common/logLevel.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/logLevel.h 2022-01-03 13:43:55.000000000 +0000 @@ -12,7 +12,6 @@ logLevelOff, logLevelAssert, logLevelError, - logLevelProtocol, logLevelWarn, logLevelInfo, logLevelDetail, diff -Nru pgbackrest-2.36/src/common/macro.h pgbackrest-2.37/src/common/macro.h --- pgbackrest-2.36/src/common/macro.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/macro.h 2022-01-03 13:43:55.000000000 +0000 @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** General Macros -Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group +Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group Portions Copyright (c) 1994, Regents of the University of California ***********************************************************************************************************************************/ #ifndef COMMON_MACRO_H diff -Nru pgbackrest-2.36/src/common/type/pack.c pgbackrest-2.37/src/common/type/pack.c --- pgbackrest-2.36/src/common/type/pack.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/type/pack.c 2022-01-03 13:43:55.000000000 +0000 @@ -553,7 +553,9 @@ if (this->tagNextTypeMap == 0xF) this->tagNextTypeMap = (unsigned int)pckReadU64Internal(this) + 0xF; - CHECK(this->tagNextTypeMap < PACK_TYPE_MAP_SIZE && packTypeMapData[this->tagNextTypeMap].type != 0); + CHECK( + FormatError, this->tagNextTypeMap < PACK_TYPE_MAP_SIZE && packTypeMapData[this->tagNextTypeMap].type != 0, + "invalid tag type"); // If the value can contain multiple bits (e.g. integer) if (packTypeMapData[this->tagNextTypeMap].valueMultiBit) @@ -1267,7 +1269,7 @@ FUNCTION_TEST_END(); ASSERT(this != NULL); - CHECK(this->tagStack.depth == 0); + CHECK(FormatError, this->tagStack.depth == 0, "invalid pack end read in container"); // Make sure we are at the end of the pack unsigned int id = UINT_MAX - 1; @@ -1463,7 +1465,7 @@ } // Else the id must be greater than the last one else - CHECK(id > this->tagStack.top->idLast); + CHECK(AssertError, id > this->tagStack.top->idLast, "field id must be greater than last id"); // Clear NULLs now that field id has been calculated this->tagStack.top->nullTotal = 0; diff -Nru pgbackrest-2.36/src/common/type/string.c pgbackrest-2.37/src/common/type/string.c --- pgbackrest-2.36/src/common/type/string.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/type/string.c 2022-01-03 13:43:55.000000000 +0000 @@ -323,7 +323,7 @@ if (requested > this->pub.extra) { // Fixed size strings may not be resized - CHECK(!STR_IS_FIXED_BUFFER()); + CHECK(AssertError, !STR_IS_FIXED_BUFFER(), "resize of fixed size string"); // Check size CHECK_SIZE(strSize(this) + requested); diff -Nru pgbackrest-2.36/src/common/type/stringId.c pgbackrest-2.37/src/common/type/stringId.c --- pgbackrest-2.36/src/common/type/stringId.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/type/stringId.c 2022-01-03 13:43:55.000000000 +0000 @@ -8,15 +8,24 @@ #include "common/type/stringId.h" /*********************************************************************************************************************************** +Number of bits to use for encoding. The number of bits affects the character set that can be encoded. +***********************************************************************************************************************************/ +typedef enum +{ + stringIdBit5 = 0, // 5-bit encoding for a-z, 2, 5, 6, and - characters + stringIdBit6 = 1, // 6-bit encoding for a-z, 0-9, A-Z, and - characters +} StringIdBit; + +/*********************************************************************************************************************************** Constants used to extract information from the header ***********************************************************************************************************************************/ -#define STRING_ID_BIT_MASK 3 #define STRING_ID_HEADER_SIZE 4 #define STRING_ID_PREFIX 4 /**********************************************************************************************************************************/ -StringId -strIdFromZN(const StringIdBit bit, const char *const buffer, const size_t size) +// Helper to do encoding for specified number of bits +static StringId +strIdBitFromZN(const StringIdBit bit, const char *const buffer, size_t size) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(ENUM, bit); @@ -61,23 +70,24 @@ break; if (map[(uint8_t)buffer[bufferIdx]] == 0) - THROW_FMT(FormatError, "'%c' is invalid for 5-bit encoding in '%s'", buffer[bufferIdx], buffer); + FUNCTION_TEST_RETURN(0); } // Set encoding in header uint64_t result = stringIdBit5; + // If size is greater than can be encoded then add prefix bit and adjust size + if (size >= STRID5_MAX) + { + result |= STRING_ID_PREFIX; + size = STRID5_MAX - 1; + } + // Encode based on the number of characters that need to be encoded switch (size) { - default: - { - // If size is greater then can be encoded set the prefix flag - if (size >= STRID5_MAX) - result |= STRING_ID_PREFIX; - + case 12: result |= (uint64_t)map[(uint8_t)buffer[11]] << 59; - } case 11: result |= (uint64_t)map[(uint8_t)buffer[10]] << 54; @@ -149,23 +159,24 @@ break; if (map[(uint8_t)buffer[bufferIdx]] == 0) - THROW_FMT(FormatError, "'%c' is invalid for 6-bit encoding in '%s'", buffer[bufferIdx], buffer); + FUNCTION_TEST_RETURN(0); } // Set encoding in header uint64_t result = stringIdBit6; + // If size is greater than can be encoded then add prefix bit and adjust size + if (size >= STRID6_MAX) + { + result |= STRING_ID_PREFIX; + size = STRID6_MAX - 1; + } + // Encode based on the number of characters that need to be encoded switch (size) { - default: - { - // If size is greater then can be encoded set the prefix flag - if (size >= STRID6_MAX) - result |= STRING_ID_PREFIX; - + case 10: result |= (uint64_t)map[(uint8_t)buffer[9]] << 58; - } case 9: result |= (uint64_t)map[(uint8_t)buffer[8]] << 52; @@ -200,6 +211,30 @@ } } +StringId +strIdFromZN(const char *const buffer, const size_t size, const bool error) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(VOID, buffer); + FUNCTION_TEST_PARAM(SIZE, size); + FUNCTION_TEST_PARAM(BOOL, error); + FUNCTION_TEST_END(); + + StringId result = strIdBitFromZN(stringIdBit5, buffer, size); + + // If 5-bit encoding fails try 6-bit + if (result == 0) + { + result = strIdBitFromZN(stringIdBit6, buffer, size); + + // Error when 6-bit encoding also fails + if (result == 0 && error) + THROW_FMT(FormatError, "'%s' contains invalid characters", buffer); + } + + FUNCTION_TEST_RETURN(result); +} + /**********************************************************************************************************************************/ size_t strIdToZN(StringId strId, char *const buffer) @@ -268,7 +303,7 @@ // 6-bit decoding default: { - CHECK(bit == stringIdBit6); + ASSERT(bit == stringIdBit6); // Map to convert encoding to characters const char map[64] = "!abcdefghijklmnopqrstuvwxyz-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; diff -Nru pgbackrest-2.36/src/common/type/stringId.h pgbackrest-2.37/src/common/type/stringId.h --- pgbackrest-2.36/src/common/type/stringId.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/common/type/stringId.h 2022-01-03 13:43:55.000000000 +0000 @@ -51,15 +51,6 @@ typedef uint64_t StringId; /*********************************************************************************************************************************** -Number of bits to use for encoding. The number of bits affects the character set that can be encoded. -***********************************************************************************************************************************/ -typedef enum -{ - stringIdBit5 = 0, // 5-bit encoding for a-z, 2, 5, 6, and - characters - stringIdBit6 = 1, // 6-bit encoding for a-z, 0-9, A-Z, and - characters -} StringIdBit; - -/*********************************************************************************************************************************** Macros to define constant StringIds. ALWAYS use bldStrId() to create these macros. The parameters in the macros are not verified against each other so the str parameter is included only for documentation purposes. ***********************************************************************************************************************************/ @@ -72,20 +63,20 @@ // Convert N chars to StringId, If the string is longer than the allowable length for the selected encoding then the StringID will // be marked as "partial" and will have a '+' appended whenever it is converted back to a string. This is to distinguish it from a // string with the same number of encoded characters that did not overflow. -StringId strIdFromZN(const StringIdBit bit, const char *const buffer, const size_t size); +StringId strIdFromZN(const char *buffer, size_t size, bool error); // Convert String to StringId using strIdFromZN() __attribute__((always_inline)) static inline StringId -strIdFromStr(const StringIdBit bit, const String *const str) +strIdFromStr(const String *const str) { - return strIdFromZN(bit, strZ(str), strSize(str)); + return strIdFromZN(strZ(str), strSize(str), true); } -// Convert zero-terminted string to StringId using strIdFromZN() +// Convert zero-terminated string to StringId using strIdFromZN() __attribute__((always_inline)) static inline StringId -strIdFromZ(const StringIdBit bit, const char *const str) +strIdFromZ(const char *const str) { - return strIdFromZN(bit, str, strlen(str)); + return strIdFromZN(str, strlen(str), true); } // Write StringId to characters without zero-terminating. The buffer at ptr must have enough space to write the entire StringId, diff -Nru pgbackrest-2.36/src/config/common.c pgbackrest-2.37/src/config/common.c --- pgbackrest-2.36/src/config/common.c 1970-01-01 00:00:00.000000000 +0000 +++ pgbackrest-2.37/src/config/common.c 2022-01-03 13:43:55.000000000 +0000 @@ -0,0 +1,121 @@ +/*********************************************************************************************************************************** +Configuration Common +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "common/debug.h" +#include "common/regExp.h" +#include "common/time.h" +#include "config/common.h" + +/**********************************************************************************************************************************/ +// Helper to get the multiplier based on the qualifier +static int64_t +cfgParseSizeQualifier(const char qualifier) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(CHAR, qualifier); + FUNCTION_TEST_END(); + + int64_t result; + + switch (qualifier) + { + case 'b': + result = 1; + break; + + case 'k': + result = 1024; + break; + + case 'm': + result = 1024 * 1024; + break; + + case 'g': + result = 1024 * 1024 * 1024; + break; + + case 't': + result = 1024LL * 1024LL * 1024LL * 1024LL; + break; + + case 'p': + result = 1024LL * 1024LL * 1024LL * 1024LL * 1024LL; + break; + + default: + THROW_FMT(AssertError, "'%c' is not a valid size qualifier", qualifier); + } + + FUNCTION_TEST_RETURN(result); +} + +int64_t +cfgParseSize(const String *const value) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, value); + FUNCTION_TEST_END(); + + ASSERT(value != NULL); + + // Lowercase the value + String *valueLower = strLower(strDup(value)); + + // Match the value against possible values + if (regExpMatchOne(STRDEF("^[0-9]+(kib|kb|k|mib|mb|m|gib|gb|g|tib|tb|t|pib|pb|p|b)*$"), valueLower)) + { + // Get the character array and size + const char *strArray = strZ(valueLower); + size_t size = strSize(valueLower); + int chrPos = -1; + + // If there is a 'b' on the end, then see if the previous character is a number + if (strArray[size - 1] == 'b') + { + // If the previous character is a number, then the letter to look at is 'b' which is the last position else it is in the + // next to last position (e.g. kb - so the 'k' is the position of interest). Only need to test for <= 9 since the regex + // enforces the format. Also allow an 'i' before the 'b'. + if (strArray[size - 2] <= '9') + chrPos = (int)(size - 1); + else if (strArray[size - 2] == 'i') + chrPos = (int)(size - 3); + else + chrPos = (int)(size - 2); + } + // Else if there is no 'b' at the end but the last position is not a number then it must be one of the letters, e.g. 'k' + else if (strArray[size - 1] > '9') + chrPos = (int)(size - 1); + + int64_t multiplier = 1; + + // If a letter was found calculate multiplier, else do nothing since assumed value is already in bytes + if (chrPos != -1) + { + multiplier = cfgParseSizeQualifier(strArray[chrPos]); + + // Remove any letters + strTrunc(valueLower, chrPos); + } + + // Convert string to bytes + FUNCTION_TEST_RETURN(cvtZToInt64(strZ(valueLower)) * multiplier); + } + + THROW_FMT(FormatError, "value '%s' is not valid", strZ(value)); +} + +/**********************************************************************************************************************************/ +int64_t +cfgParseTime(const String *const value) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, value); + FUNCTION_TEST_END(); + + ASSERT(value != NULL); + + FUNCTION_TEST_RETURN((int64_t)(cvtZToDouble(strZ(value)) * MSEC_PER_SEC)); +} diff -Nru pgbackrest-2.36/src/config/common.h pgbackrest-2.37/src/config/common.h --- pgbackrest-2.36/src/config/common.h 1970-01-01 00:00:00.000000000 +0000 +++ pgbackrest-2.37/src/config/common.h 2022-01-03 13:43:55.000000000 +0000 @@ -0,0 +1,18 @@ +/*********************************************************************************************************************************** +Configuration Common +***********************************************************************************************************************************/ +#ifndef CONFIG_COMMON_H +#define CONFIG_COMMON_H + +#include "common/type/string.h" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Parse option size, e.g. 23m +int64_t cfgParseSize(const String *value); + +// Parse option time, e.g. 900 +int64_t cfgParseTime(const String *value); + +#endif diff -Nru pgbackrest-2.36/src/config/config.auto.h pgbackrest-2.37/src/config/config.auto.h --- pgbackrest-2.36/src/config/config.auto.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/config.auto.h 2022-01-03 13:43:55.000000000 +0000 @@ -22,8 +22,8 @@ #define CFGCMD_REPO_PUT "repo-put" #define CFGCMD_REPO_RM "repo-rm" #define CFGCMD_RESTORE "restore" +#define CFGCMD_SERVER "server" #define CFGCMD_SERVER_PING "server-ping" -#define CFGCMD_SERVER_START "server-start" #define CFGCMD_STANZA_CREATE "stanza-create" #define CFGCMD_STANZA_DELETE "stanza-delete" #define CFGCMD_STANZA_UPGRADE "stanza-upgrade" @@ -55,6 +55,7 @@ #define CFGOPT_BUFFER_SIZE "buffer-size" #define CFGOPT_CHECKSUM_PAGE "checksum-page" #define CFGOPT_CIPHER_PASS "cipher-pass" +#define CFGOPT_CMD "cmd" #define CFGOPT_CMD_SSH "cmd-ssh" #define CFGOPT_COMPRESS "compress" #define CFGOPT_COMPRESS_LEVEL "compress-level" @@ -125,7 +126,7 @@ #define CFGOPT_TLS_SERVER_PORT "tls-server-port" #define CFGOPT_TYPE "type" -#define CFG_OPTION_TOTAL 148 +#define CFG_OPTION_TOTAL 149 /*********************************************************************************************************************************** Option value constants @@ -325,8 +326,8 @@ cfgCmdRepoPut, cfgCmdRepoRm, cfgCmdRestore, + cfgCmdServer, cfgCmdServerPing, - cfgCmdServerStart, cfgCmdStanzaCreate, cfgCmdStanzaDelete, cfgCmdStanzaUpgrade, @@ -364,6 +365,7 @@ cfgOptBufferSize, cfgOptChecksumPage, cfgOptCipherPass, + cfgOptCmd, cfgOptCmdSsh, cfgOptCompress, cfgOptCompressLevel, diff -Nru pgbackrest-2.36/src/config/config.c pgbackrest-2.37/src/config/config.c --- pgbackrest-2.36/src/config/config.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/config.c 2022-01-03 13:43:55.000000000 +0000 @@ -456,6 +456,10 @@ { FUNCTION_TEST_RETURN(strNewDbl((double)varInt64(value) / MSEC_PER_SEC)); } + else if (optionType == cfgOptTypeStringId) + { + FUNCTION_TEST_RETURN(strIdToStr(varUInt64(value))); + } FUNCTION_TEST_RETURN(varStrForce(value)); } @@ -773,6 +777,9 @@ case cfgOptDataTypeList: FUNCTION_TEST_RETURN(varNewVarLst(configLocal->option[optionId].index[optionIdx].value.list)); + case cfgOptDataTypeStringId: + FUNCTION_TEST_RETURN(varNewUInt64(configLocal->option[optionId].index[optionIdx].value.stringId)); + default: ASSERT(configLocal->option[optionId].dataType == cfgOptDataTypeString); break; @@ -916,34 +923,6 @@ FUNCTION_LOG_RETURN_CONST(STRING, cfgOptionIdxInternal(optionId, optionIdx, cfgOptDataTypeString, true)->value.string); } -// Helper to convert option String values to StringIds. Some options need 6-bit encoding while most work fine with 5-bit encoding. -// At some point the config parser will work with StringIds directly and this code can be removed, but for now it protects the -// callers from this logic and hopefully means no changes to the callers when the parser is updated. -static StringId -cfgOptionStrIdInternal( - const ConfigOption optionId, const unsigned int optionIdx) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(ENUM, optionId); - FUNCTION_TEST_PARAM(UINT, optionIdx); - FUNCTION_TEST_END(); - - const String *const value = cfgOptionIdxInternal(optionId, optionIdx, cfgOptDataTypeString, false)->value.string; - StringId result = 0; - - TRY_BEGIN() - { - result = strIdFromStr(stringIdBit5, value); - } - CATCH_ANY() - { - result = strIdFromStr(stringIdBit6, value); - } - TRY_END(); - - FUNCTION_TEST_RETURN(result); -} - StringId cfgOptionIdxStrId(ConfigOption optionId, unsigned int optionIdx) { @@ -952,7 +931,7 @@ FUNCTION_LOG_PARAM(UINT, optionIdx); FUNCTION_LOG_END(); - FUNCTION_LOG_RETURN(STRING_ID, cfgOptionStrIdInternal(optionId, optionIdx)); + FUNCTION_LOG_RETURN(STRING_ID, cfgOptionIdxInternal(optionId, optionIdx, cfgOptDataTypeStringId, false)->value.stringId); } /**********************************************************************************************************************************/ @@ -1021,6 +1000,16 @@ break; } + + case cfgOptDataTypeStringId: + { + if (varType(value) == varTypeUInt64) + configLocal->option[optionId].index[optionIdx].value.stringId = varUInt64(value); + else + configLocal->option[optionId].index[optionIdx].value.stringId = strIdFromStr(varStr(value)); + + break; + } default: THROW_FMT(AssertError, "set not available for option data type %u", configLocal->option[optionId].dataType); diff -Nru pgbackrest-2.36/src/config/config.intern.h pgbackrest-2.37/src/config/config.intern.h --- pgbackrest-2.36/src/config/config.intern.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/config.intern.h 2022-01-03 13:43:55.000000000 +0000 @@ -28,6 +28,7 @@ const KeyValue *keyValue; // KeyValue const VariantList *list; // VariantList const String *string; // String + StringId stringId; // StringId } ConfigOptionValueType; typedef struct ConfigOptionValue diff -Nru pgbackrest-2.36/src/config/exec.c pgbackrest-2.37/src/config/exec.c --- pgbackrest-2.36/src/config/exec.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/exec.c 2022-01-03 13:43:55.000000000 +0000 @@ -114,7 +114,11 @@ else { valueList = strLstNew(); - strLstAdd(valueList, cfgOptionDisplayVar(value, cfgParseOptionType(optionId))); + + strLstAdd( + valueList, + exists ? cfgOptionDisplayVar(value, cfgParseOptionType(optionId)) : + cfgOptionIdxDisplay(optionId, optionIdx)); } // Output options and values diff -Nru pgbackrest-2.36/src/config/load.c pgbackrest-2.37/src/config/load.c --- pgbackrest-2.36/src/config/load.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/load.c 2022-01-03 13:43:55.000000000 +0000 @@ -40,15 +40,13 @@ unsigned int logProcessMax = 1; if (cfgOptionValid(cfgOptLogLevelConsole)) - logLevelConsole = logLevelEnum(strZ(cfgOptionStr(cfgOptLogLevelConsole))); + logLevelConsole = logLevelEnum(cfgOptionStrId(cfgOptLogLevelConsole)); if (cfgOptionValid(cfgOptLogLevelStderr)) - { - logLevelStdErr = logLevelEnum(strZ(cfgOptionStr(cfgOptLogLevelStderr))); - } + logLevelStdErr = logLevelEnum(cfgOptionStrId(cfgOptLogLevelStderr)); if (cfgOptionValid(cfgOptLogLevelFile)) - logLevelFile = logLevelEnum(strZ(cfgOptionStr(cfgOptLogLevelFile))); + logLevelFile = logLevelEnum(cfgOptionStrId(cfgOptLogLevelFile)); if (cfgOptionValid(cfgOptLogTimestamp)) logTimestamp = cfgOptionBool(cfgOptLogTimestamp); @@ -95,7 +93,7 @@ for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++) { if (optionIdx != repoIdx && !(cfgOptionIdxTest(cfgOptRepoHost, repoIdx)) && - strEq(cfgOptionIdxStr(cfgOptRepoType, optionIdx), cfgOptionIdxStr(cfgOptRepoType, repoIdx)) && + cfgOptionIdxStrId(cfgOptRepoType, optionIdx) == cfgOptionIdxStrId(cfgOptRepoType, repoIdx) && strEq(cfgOptionIdxStr(cfgOptRepoPath, optionIdx), cfgOptionIdxStr(cfgOptRepoPath, repoIdx))) { THROW_FMT( @@ -109,6 +107,10 @@ } } + // Set default for cmd + if (cfgOptionValid(cfgOptCmd)) + cfgOptionDefaultSet(cfgOptCmd, VARSTR(cfgExe())); + // Set default for repo-host-cmd if (cfgOptionValid(cfgOptRepoHostCmd)) cfgOptionDefaultSet(cfgOptRepoHostCmd, VARSTR(cfgExe())); @@ -329,7 +331,7 @@ // Set compress-type to none. Eventually the compress option will be deprecated and removed so this reduces code churn // when that happens. if (!cfgOptionBool(cfgOptCompress) && cfgOptionSource(cfgOptCompressType) == cfgSourceDefault) - cfgOptionSet(cfgOptCompressType, cfgSourceParam, VARSTR(compressTypeStr(compressTypeNone))); + cfgOptionSet(cfgOptCompressType, cfgSourceParam, VARUINT64(CFGOPTVAL_COMPRESS_TYPE_NONE)); } // Now invalidate compress so it can't be used and won't be passed to child processes @@ -339,14 +341,14 @@ // Check that selected compress type has been compiled into this binary if (cfgOptionValid(cfgOptCompressType)) - compressTypePresent(compressTypeEnum(cfgOptionStr(cfgOptCompressType))); + compressTypePresent(compressTypeEnum(cfgOptionStrId(cfgOptCompressType))); // Update compress-level default based on the compression type if (cfgOptionValid(cfgOptCompressLevel) && cfgOptionSource(cfgOptCompressLevel) == cfgSourceDefault) { cfgOptionSet( cfgOptCompressLevel, cfgSourceDefault, - VARINT64(compressLevelDefault(compressTypeEnum(cfgOptionStr(cfgOptCompressType))))); + VARINT64(compressLevelDefault(compressTypeEnum(cfgOptionStrId(cfgOptCompressType))))); } FUNCTION_LOG_RETURN_VOID(); @@ -387,7 +389,7 @@ // Attempt to open log file if (!logFileSet(strZ(logFile))) - cfgOptionSet(cfgOptLogLevelFile, cfgSourceParam, varNewStrZ("off")); + cfgOptionSet(cfgOptLogLevelFile, cfgSourceParam, VARUINT64(CFGOPTVAL_LOG_LEVEL_CONSOLE_OFF)); } MEM_CONTEXT_TEMP_END(); } diff -Nru pgbackrest-2.36/src/config/parse.auto.c pgbackrest-2.37/src/config/parse.auto.c --- pgbackrest-2.36/src/config/parse.auto.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/parse.auto.c 2022-01-03 13:43:55.000000000 +0000 @@ -16,12 +16,12 @@ PARSE_RULE_STRPUB("/var/log/pgbackrest"), PARSE_RULE_STRPUB("/var/spool/pgbackrest"), PARSE_RULE_STRPUB("1"), - PARSE_RULE_STRPUB("1048576"), - PARSE_RULE_STRPUB("1073741824"), - PARSE_RULE_STRPUB("134217728"), + PARSE_RULE_STRPUB("128MiB"), PARSE_RULE_STRPUB("15"), PARSE_RULE_STRPUB("1800"), PARSE_RULE_STRPUB("1830"), + PARSE_RULE_STRPUB("1GiB"), + PARSE_RULE_STRPUB("1MiB"), PARSE_RULE_STRPUB("2"), PARSE_RULE_STRPUB("3"), PARSE_RULE_STRPUB("443"), @@ -63,12 +63,12 @@ parseRuleValStrQT_FS_var_FS_log_FS_pgbackrest_QT, parseRuleValStrQT_FS_var_FS_spool_FS_pgbackrest_QT, parseRuleValStrQT_1_QT, - parseRuleValStrQT_1048576_QT, - parseRuleValStrQT_1073741824_QT, - parseRuleValStrQT_134217728_QT, + parseRuleValStrQT_128MiB_QT, parseRuleValStrQT_15_QT, parseRuleValStrQT_1800_QT, parseRuleValStrQT_1830_QT, + parseRuleValStrQT_1GiB_QT, + parseRuleValStrQT_1MiB_QT, parseRuleValStrQT_2_QT, parseRuleValStrQT_3_QT, parseRuleValStrQT_443_QT, @@ -521,10 +521,10 @@ // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_COMMAND ( - PARSE_RULE_COMMAND_NAME("server-ping"), + PARSE_RULE_COMMAND_NAME("server"), PARSE_RULE_COMMAND_LOCK_TYPE(lockTypeNone), + PARSE_RULE_COMMAND_LOG_FILE(true), PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(logLevelInfo), - PARSE_RULE_COMMAND_PARAMETER_ALLOWED(true), PARSE_RULE_COMMAND_ROLE_VALID_LIST ( @@ -535,10 +535,10 @@ // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_COMMAND ( - PARSE_RULE_COMMAND_NAME("server-start"), + PARSE_RULE_COMMAND_NAME("server-ping"), PARSE_RULE_COMMAND_LOCK_TYPE(lockTypeNone), - PARSE_RULE_COMMAND_LOG_FILE(true), PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(logLevelInfo), + PARSE_RULE_COMMAND_PARAMETER_ALLOWED(true), PARSE_RULE_COMMAND_ROLE_VALID_LIST ( @@ -848,7 +848,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( PARSE_RULE_VAL_INT(parseRuleValInt134217728), - PARSE_RULE_VAL_STR(parseRuleValStrQT_134217728_QT), + PARSE_RULE_VAL_STR(parseRuleValStrQT_128MiB_QT), ), ), ), @@ -890,7 +890,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("archive-mode"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -912,6 +912,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdPreserve), PARSE_RULE_VAL_STR(parseRuleValStrQT_preserve_QT), ), ), @@ -1073,8 +1074,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -1137,7 +1138,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( PARSE_RULE_VAL_INT(parseRuleValInt1048576), - PARSE_RULE_VAL_STR(parseRuleValStrQT_1048576_QT), + PARSE_RULE_VAL_STR(parseRuleValStrQT_1MiB_QT), ), ), ), @@ -1177,6 +1178,50 @@ // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_OPTION ( + PARSE_RULE_OPTION_NAME("cmd"), + PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_RESET(true), + PARSE_RULE_OPTION_REQUIRED(false), + PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), + + PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) + ), + + PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + ), + + PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST + ( + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) + ), + ), + + // ----------------------------------------------------------------------------------------------------------------------------- + PARSE_RULE_OPTION + ( PARSE_RULE_OPTION_NAME("cmd-ssh"), PARSE_RULE_OPTION_TYPE(cfgOptTypeString), PARSE_RULE_OPTION_RESET(true), @@ -1354,7 +1399,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("compress-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -1385,6 +1430,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdGz), PARSE_RULE_VAL_STR(parseRuleValStrQT_gz_QT), ), ), @@ -1414,8 +1460,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -1492,8 +1538,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -1570,8 +1616,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -1822,8 +1868,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2004,8 +2050,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2320,7 +2366,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("log-level-console"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -2339,8 +2385,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2400,6 +2446,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdWarn), PARSE_RULE_VAL_STR(parseRuleValStrQT_warn_QT), ), ), @@ -2410,7 +2457,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("log-level-file"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -2429,8 +2476,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2490,6 +2537,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdInfo), PARSE_RULE_VAL_STR(parseRuleValStrQT_info_QT), ), ), @@ -2500,7 +2548,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("log-level-stderr"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -2519,8 +2567,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2580,6 +2628,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdWarn), PARSE_RULE_VAL_STR(parseRuleValStrQT_warn_QT), ), ), @@ -2609,8 +2658,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2764,8 +2813,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -2847,7 +2896,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( PARSE_RULE_VAL_INT(parseRuleValInt1073741824), - PARSE_RULE_VAL_STR(parseRuleValStrQT_1073741824_QT), + PARSE_RULE_VAL_STR(parseRuleValStrQT_1GiB_QT), ), ), ), @@ -2961,7 +3010,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("output"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionCommandLine), @@ -2983,6 +3032,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdText), PARSE_RULE_VAL_STR(parseRuleValStrQT_text_QT), ), ), @@ -3526,7 +3576,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("pg-host-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -3577,6 +3627,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdSsh), PARSE_RULE_VAL_STR(parseRuleValStrQT_ssh_QT), ), ), @@ -4002,7 +4053,7 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -4152,7 +4203,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("remote-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_REQUIRED(false), PARSE_RULE_OPTION_SECTION(cfgSectionCommandLine), @@ -4585,7 +4636,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-azure-key-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -4663,6 +4714,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdShared), PARSE_RULE_VAL_STR(parseRuleValStrQT_shared_QT), ), ), @@ -4673,7 +4725,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-azure-uri-style"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -4751,6 +4803,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdHost), PARSE_RULE_VAL_STR(parseRuleValStrQT_host_QT), ), ), @@ -4840,7 +4893,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-cipher-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -4913,6 +4966,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdNone), PARSE_RULE_VAL_STR(parseRuleValStrQT_none_QT), ), ), @@ -5159,7 +5213,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-gcs-key-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -5238,6 +5292,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdService), PARSE_RULE_VAL_STR(parseRuleValStrQT_service_QT), ), ), @@ -5878,7 +5933,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-host-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -5938,6 +5993,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdSsh), PARSE_RULE_VAL_STR(parseRuleValStrQT_ssh_QT), ), ), @@ -6176,7 +6232,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-retention-archive-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -6202,6 +6258,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdFull), PARSE_RULE_VAL_STR(parseRuleValStrQT_full_QT), ), ), @@ -6272,7 +6329,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-retention-full-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -6297,6 +6354,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdCount), PARSE_RULE_VAL_STR(parseRuleValStrQT_count_QT), ), ), @@ -6651,7 +6709,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-s3-key-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -6730,6 +6788,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdShared), PARSE_RULE_VAL_STR(parseRuleValStrQT_shared_QT), ), ), @@ -6973,7 +7032,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-s3-uri-style"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -7051,6 +7110,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdHost), PARSE_RULE_VAL_STR(parseRuleValStrQT_host_QT), ), ), @@ -7472,7 +7532,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("repo-type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_RESET(true), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), @@ -7548,6 +7608,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdPosix), PARSE_RULE_VAL_STR(parseRuleValStrQT_posix_QT), ), ), @@ -7605,8 +7666,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -7683,8 +7744,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -7789,7 +7850,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("sort"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionCommandLine), @@ -7811,6 +7872,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdAsc), PARSE_RULE_VAL_STR(parseRuleValStrQT_asc_QT), ), ), @@ -8081,7 +8143,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("target-action"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionCommandLine), @@ -8113,6 +8175,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdPause), PARSE_RULE_VAL_STR(parseRuleValStrQT_pause_QT), ), ), @@ -8206,8 +8269,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -8290,8 +8353,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -8374,8 +8437,8 @@ PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) @@ -8446,8 +8509,8 @@ PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST ( + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) ), PARSE_RULE_OPTIONAL @@ -8474,7 +8537,7 @@ PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST ( - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) ), ), @@ -8489,7 +8552,7 @@ PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST ( - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) ), ), @@ -8504,7 +8567,7 @@ PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST ( - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) ), ), @@ -8519,7 +8582,7 @@ PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST ( - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) ), ), @@ -8534,8 +8597,8 @@ PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST ( + PARSE_RULE_OPTION_COMMAND(cfgCmdServer) PARSE_RULE_OPTION_COMMAND(cfgCmdServerPing) - PARSE_RULE_OPTION_COMMAND(cfgCmdServerStart) ), PARSE_RULE_OPTIONAL @@ -8561,7 +8624,7 @@ PARSE_RULE_OPTION ( PARSE_RULE_OPTION_NAME("type"), - PARSE_RULE_OPTION_TYPE(cfgOptTypeString), + PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId), PARSE_RULE_OPTION_REQUIRED(true), PARSE_RULE_OPTION_SECTION(cfgSectionCommandLine), @@ -8589,6 +8652,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdIncr), PARSE_RULE_VAL_STR(parseRuleValStrQT_incr_QT), ), ), @@ -8615,6 +8679,7 @@ PARSE_RULE_OPTIONAL_DEFAULT ( + PARSE_RULE_VAL_STRID(parseRuleValStrIdDefault), PARSE_RULE_VAL_STR(parseRuleValStrQT_default_QT), ), ), @@ -8931,6 +8996,7 @@ cfgOptBufferSize, cfgOptChecksumPage, cfgOptCipherPass, + cfgOptCmd, cfgOptCmdSsh, cfgOptCompress, cfgOptCompressLevel, diff -Nru pgbackrest-2.36/src/config/parse.c pgbackrest-2.37/src/config/parse.c --- pgbackrest-2.36/src/config/parse.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/parse.c 2022-01-03 13:43:55.000000000 +0000 @@ -16,6 +16,7 @@ #include "common/macro.h" #include "common/memContext.h" #include "common/regExp.h" +#include "config/common.h" #include "config/config.intern.h" #include "config/parse.h" #include "version.h" @@ -117,7 +118,7 @@ typedef struct ParseRuleOption { const char *name; // Name - unsigned int type:3; // e.g. string, int, boolean + unsigned int type:4; // e.g. string, int, boolean bool negate:1; // Can the option be negated on the command line? bool reset:1; // Can the option be reset on the command line? bool required:1; // Is the option required? @@ -288,7 +289,7 @@ parseOptionIdxValue(ParseOption *optionList, unsigned int optionId, unsigned int optionKeyIdx) { FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(PARSE_OPTION, parseOption); // Structure containing all options being parsed + FUNCTION_TEST_PARAM_P(PARSE_OPTION, optionList); // Structure containing all options being parsed FUNCTION_TEST_PARAM(UINT, optionId); // Unique ID which also identifies the option in the parse list FUNCTION_TEST_PARAM(UINT, optionKeyIdx); // Zero-based key index (e.g. pg3-path => 2), 0 for non-indexed FUNCTION_TEST_END(); @@ -687,6 +688,9 @@ case cfgOptTypeList: FUNCTION_TEST_RETURN(cfgOptDataTypeList); + case cfgOptTypeStringId: + FUNCTION_TEST_RETURN(cfgOptDataTypeStringId); + default: break; } @@ -747,21 +751,6 @@ // If a depend list exists, make sure the value is in the list if (pckReadNext(filter)) { - StringId dependValueStrId = 0; - - if (cfgParseOptionDataType(dependId) == cfgOptDataTypeString) - { - TRY_BEGIN() - { - dependValueStrId = strIdFromStr(stringIdBit5, dependValue->value.string); - } - CATCH_ANY() - { - dependValueStrId = strIdFromStr(stringIdBit6, dependValue->value.string); - } - TRY_END(); - } - do { switch (cfgParseOptionDataType(dependId)) @@ -772,9 +761,9 @@ default: { - ASSERT(cfgParseOptionDataType(dependId) == cfgOptDataTypeString); + ASSERT(cfgParseOptionDataType(dependId) == cfgOptDataTypeStringId); - if (parseRuleValueStrId[pckReadU32P(filter)] == dependValueStrId) + if (parseRuleValueStrId[pckReadU32P(filter)] == dependValue->value.stringId) result = true; break; } @@ -974,6 +963,11 @@ break; } + + case cfgOptTypeStringId: + optionalRules->defaultValue.stringId = parseRuleValueStrId[pckReadU32P(ruleData)]; + optionalRules->defaultRaw = (const String *)&parseRuleValueStr[pckReadU32P(ruleData)]; + break; } } } @@ -1150,104 +1144,6 @@ } /*********************************************************************************************************************************** -Generate multiplier based on character -***********************************************************************************************************************************/ -static uint64_t -sizeQualifierToMultiplier(char qualifier) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(CHAR, qualifier); - FUNCTION_TEST_END(); - - uint64_t result; - - switch (qualifier) - { - case 'b': - result = 1; - break; - - case 'k': - result = 1024; - break; - - case 'm': - result = 1024 * 1024; - break; - - case 'g': - result = 1024 * 1024 * 1024; - break; - - case 't': - result = 1024LL * 1024LL * 1024LL * 1024LL; - break; - - case 'p': - result = 1024LL * 1024LL * 1024LL * 1024LL * 1024LL; - break; - - default: - THROW_FMT(AssertError, "'%c' is not a valid size qualifier", qualifier); - } - - FUNCTION_TEST_RETURN(result); -} - -static uint64_t -convertToByte(const String *value) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, value); - FUNCTION_TEST_END(); - - ASSERT(value != NULL); - - // Lowercase the value - String *valueLower = strLower(strDup(value)); - - // Match the value against possible values - if (regExpMatchOne(STRDEF("^[0-9]+(kb|k|mb|m|gb|g|tb|t|pb|p|b)*$"), valueLower)) - { - // Get the character array and size - const char *strArray = strZ(valueLower); - size_t size = strSize(valueLower); - int chrPos = -1; - - // If there is a 'b' on the end, then see if the previous character is a number - if (strArray[size - 1] == 'b') - { - // If the previous character is a number, then the letter to look at is 'b' which is the last position else it is in the - // next to last position (e.g. kb - so the 'k' is the position of interest). Only need to test for <= 9 since the regex - // enforces the format. - if (strArray[size - 2] <= '9') - chrPos = (int)(size - 1); - else - chrPos = (int)(size - 2); - } - // else if there is no 'b' at the end but the last position is not a number then it must be one of the letters, e.g. 'k' - else if (strArray[size - 1] > '9') - chrPos = (int)(size - 1); - - uint64_t multiplier = 1; - - // If a letter was found calculate multiplier, else do nothing since assumed value is already in bytes - if (chrPos != -1) - { - multiplier = sizeQualifierToMultiplier(strArray[chrPos]); - - // Remove any letters - strTrunc(valueLower, chrPos); - } - - // Convert string to bytes - FUNCTION_TEST_RETURN(cvtZToUInt64(strZ(valueLower)) * multiplier); - } - else - THROW_FMT(FormatError, "value '%s' is not valid", strZ(value)); -} - -/*********************************************************************************************************************************** Load the configuration file(s) The parent mem context is used. Defaults are passed to make testing easier. @@ -1515,11 +1411,26 @@ if (!option.found) THROW_FMT(OptionInvalidError, "invalid option '--%s'", arg); - // If the option requires an argument - if (parseRuleOption[option.id].type != cfgOptTypeBoolean && !option.negate && !option.reset) + // If the option may have an argument (arguments are optional for boolean options) + if (!option.negate && !option.reset) { + // Handle boolean (only y/n allowed as argument) + if (parseRuleOption[option.id].type == cfgOptTypeBoolean) + { + // Validate argument/set negate when argument present + if (optionArg != NULL) + { + if (strEqZ(optionArg, "n")) + option.negate = true; + else if (!strEqZ(optionArg, "y")) + { + THROW_FMT( + OptionInvalidValueError, "boolean option '--%s' argument must be 'y' or 'n'", strZ(optionName)); + } + } + } // If no argument was found with the option then try the next argument - if (optionArg == NULL) + else if (optionArg == NULL) { // Error if there are no more arguments in the list if (argListIdx == argListSize - 1) @@ -2138,7 +2049,7 @@ default: { - ASSERT(cfgParseOptionDataType(dependId) == cfgOptDataTypeString); + ASSERT(cfgParseOptionDataType(dependId) == cfgOptDataTypeStringId); String *const errorList = strNew(); unsigned int validSize = 0; @@ -2247,16 +2158,14 @@ break; case cfgOptTypeSize: - configOptionValue->value.integer = (int64_t)convertToByte(value); - valueAllow = varStrForce(VARINT64(configOptionValue->value.integer)); + configOptionValue->value.integer = cfgParseSize(value); break; default: { ASSERT(optionType == cfgOptTypeTime); - configOptionValue->value.integer = (int64_t)(cvtZToDouble( - strZ(value)) * MSEC_PER_SEC); + configOptionValue->value.integer = cfgParseTime(value); break; } } @@ -2279,6 +2188,11 @@ cfgParseOptionKeyIdxName(optionId, optionKeyIdx)); } } + // Else if StringId + else if (optionType == cfgOptTypeStringId) + { + configOptionValue->value.stringId = strIdFromZN(strZ(valueAllow), strSize(valueAllow), false); + } // Else if string make sure it is valid else { @@ -2336,38 +2250,14 @@ PackRead *const allowList = pckReadNewC(optionalRules.allowList, optionalRules.allowListSize); bool allowListFound = false; - if (parseRuleOption[optionId].type == cfgOptTypeString) + if (parseRuleOption[optionId].type == cfgOptTypeStringId) { - bool valueValid = true; - StringId value = 0; - - TRY_BEGIN() + while (pckReadNext(allowList)) { - TRY_BEGIN() + if (parseRuleValueStrId[pckReadU32P(allowList)] == configOptionValue->value.stringId) { - value = strIdFromStr(stringIdBit5, valueAllow); - } - CATCH_ANY() - { - value = strIdFromStr(stringIdBit6, valueAllow); - } - TRY_END(); - } - CATCH_ANY() - { - valueValid = false; - } - TRY_END(); - - if (valueValid) - { - while (pckReadNext(allowList)) - { - if (parseRuleValueStrId[pckReadU32P(allowList)] == value) - { - allowListFound = true; - break; - } + allowListFound = true; + break; } } } diff -Nru pgbackrest-2.36/src/config/parse.h pgbackrest-2.37/src/config/parse.h --- pgbackrest-2.36/src/config/parse.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/parse.h 2022-01-03 13:43:55.000000000 +0000 @@ -19,6 +19,7 @@ cfgOptTypePath, // Path string with validation cfgOptTypeSize, // Size, e.g. 1m, 2gb cfgOptTypeString, // String + cfgOptTypeStringId, // StringId cfgOptTypeTime, // Time in seconds, e.g. 23, 1.5 } ConfigOptionType; @@ -32,6 +33,7 @@ cfgOptDataTypeInteger, // Signed 64-bit integer cfgOptDataTypeList, // List cfgOptDataTypeString, // String + cfgOptDataTypeStringId, // StringId } ConfigOptionDataType; /*********************************************************************************************************************************** diff -Nru pgbackrest-2.36/src/config/protocol.c pgbackrest-2.37/src/config/protocol.c --- pgbackrest-2.36/src/config/protocol.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/config/protocol.c 2022-01-03 13:43:55.000000000 +0000 @@ -31,7 +31,7 @@ while (pckReadNext(param)) { CfgParseOptionResult option = cfgParseOptionP(pckReadStrP(param)); - CHECK(option.found); + CHECK(AssertError, option.found, "option not found"); varLstAdd(optionList, varDup(cfgOptionIdxVar(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1)))); } diff -Nru pgbackrest-2.36/src/configure pgbackrest-2.37/src/configure --- pgbackrest-2.36/src/configure 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/configure 2022-01-03 13:43:55.000000000 +0000 @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for pgBackRest 2.36. +# Generated by GNU Autoconf 2.69 for pgBackRest 2.37. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,8 +577,8 @@ # Identity of this package. PACKAGE_NAME='pgBackRest' PACKAGE_TARNAME='pgbackrest' -PACKAGE_VERSION='2.36' -PACKAGE_STRING='pgBackRest 2.36' +PACKAGE_VERSION='2.37' +PACKAGE_STRING='pgBackRest 2.37' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -1252,7 +1252,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures pgBackRest 2.36 to adapt to many kinds of systems. +\`configure' configures pgBackRest 2.37 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1299,7 +1299,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of pgBackRest 2.36:";; + short | recursive ) echo "Configuration of pgBackRest 2.37:";; esac cat <<\_ACEOF @@ -1394,7 +1394,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -pgBackRest configure 2.36 +pgBackRest configure 2.37 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1692,7 +1692,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by pgBackRest $as_me 2.36, which was +It was created by pgBackRest $as_me 2.37, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4906,7 +4906,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by pgBackRest $as_me 2.36, which was +This file was extended by pgBackRest $as_me 2.37, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -4968,7 +4968,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -pgBackRest config.status 2.36 +pgBackRest config.status 2.37 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -5672,4 +5672,4 @@ $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} fi -# Generated from src/build/configure.ac sha1 78d7e6c49cc5f7ee24cc6fed9e85e524fdb8145e +# Generated from src/build/configure.ac sha1 dd9ff5ccb512474713d768dd08f97e2d90b308f1 diff -Nru pgbackrest-2.36/src/db/db.c pgbackrest-2.37/src/db/db.c --- pgbackrest-2.36/src/db/db.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/db/db.c 2022-01-03 13:43:55.000000000 +0000 @@ -7,6 +7,8 @@ #include "common/log.h" #include "common/type/json.h" #include "common/wait.h" +#include "config/config.h" +#include "config/protocol.h" #include "db/db.h" #include "db/protocol.h" #include "postgres/interface.h" @@ -18,6 +20,7 @@ Constants ***********************************************************************************************************************************/ #define PG_BACKUP_ADVISORY_LOCK "12340078987004321" +#define DB_PING_SEC 30 /*********************************************************************************************************************************** Object type @@ -28,7 +31,9 @@ PgClient *client; // Local PostgreSQL client ProtocolClient *remoteClient; // Protocol client for remote db queries unsigned int remoteIdx; // Index provided by the remote on open for subsequent calls + const Storage *storage; // PostgreSQL storage const String *applicationName; // Used to identify this connection in PostgreSQL + time_t pingTimeLast; // Last time cluster was pinged }; /*********************************************************************************************************************************** @@ -55,15 +60,17 @@ /**********************************************************************************************************************************/ Db * -dbNew(PgClient *client, ProtocolClient *remoteClient, const String *applicationName) +dbNew(PgClient *client, ProtocolClient *remoteClient, const Storage *const storage, const String *applicationName) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(PG_CLIENT, client); FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, remoteClient); + FUNCTION_LOG_PARAM(STORAGE, storage); FUNCTION_LOG_PARAM(STRING, applicationName); FUNCTION_LOG_END(); ASSERT((client != NULL && remoteClient == NULL) || (client == NULL && remoteClient != NULL)); + ASSERT(storage != NULL); ASSERT(applicationName != NULL); Db *this = NULL; @@ -79,6 +86,7 @@ .memContext = memContextCurrent(), }, .remoteClient = remoteClient, + .storage = storage, .applicationName = strDup(applicationName), }; @@ -137,7 +145,7 @@ ASSERT(this != NULL); ASSERT(command != NULL); - CHECK(dbQuery(this, command) == NULL); + CHECK(AssertError, dbQuery(this, command) == NULL, "exec returned data"); FUNCTION_LOG_RETURN_VOID(); } @@ -158,8 +166,8 @@ VariantList *result = dbQuery(this, query); - CHECK(varLstSize(result) == 1); - CHECK(varLstSize(varVarLst(varLstGet(result, 0))) == 1); + CHECK(AssertError, varLstSize(result) == 1, "query must return one column"); + CHECK(AssertError, varLstSize(varVarLst(varLstGet(result, 0))) == 1, "query must return one row"); FUNCTION_LOG_RETURN(VARIANT, varLstGet(varVarLst(varLstGet(result, 0)), 0)); } @@ -180,11 +188,31 @@ VariantList *result = dbQuery(this, query); - CHECK(varLstSize(result) == 1); + CHECK(AssertError, varLstSize(result) == 1, "query must return one row"); FUNCTION_LOG_RETURN(VARIANT_LIST, varVarLst(varLstGet(result, 0))); } +/*********************************************************************************************************************************** +Is the cluster in recovery? +***********************************************************************************************************************************/ +static bool +dbIsInRecovery(Db *const this) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(DB, this); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + bool result = false; + + if (dbPgVersion(this) >= PG_VERSION_HOT_STANDBY) + result = varBool(dbQueryColumn(this, STRDEF("select pg_catalog.pg_is_in_recovery()"))); + + FUNCTION_LOG_RETURN(BOOL, result); +} + /**********************************************************************************************************************************/ void dbOpen(Db *this) @@ -205,9 +233,16 @@ // Set a callback to notify the remote when a connection is closed memContextCallbackSet(this->pub.memContext, dbFreeResource, this); + + // Get db-timeout from the remote since it might be different than the local value + this->pub.dbTimeout = varUInt64Force( + varLstGet(configOptionRemote(this->remoteClient, varLstAdd(varLstNew(), varNewStrZ(CFGOPT_DB_TIMEOUT))), 0)); } else + { pgClientOpen(this->client); + this->pub.dbTimeout = cfgOptionUInt64(cfgOptDbTimeout); + } // Set search_path to prevent overrides of the functions we expect to call. All queries should also be schema-qualified, // but this is an extra level protection. @@ -223,7 +258,8 @@ "select (select setting from pg_catalog.pg_settings where name = 'server_version_num')::int4," " (select setting from pg_catalog.pg_settings where name = 'data_directory')::text," " (select setting from pg_catalog.pg_settings where name = 'archive_mode')::text," - " (select setting from pg_catalog.pg_settings where name = 'archive_command')::text")); + " (select setting from pg_catalog.pg_settings where name = 'archive_command')::text," + " (select setting from pg_catalog.pg_settings where name = 'checkpoint_timeout')::int4")); // Check that none of the return values are null, which indicates the user cannot select some rows in pg_settings for (unsigned int columnIdx = 0; columnIdx < varLstSize(row); columnIdx++) @@ -244,11 +280,13 @@ // Store the data directory that PostgreSQL is running in, the archive mode, and archive command. These can be compared to // the configured pgBackRest directory, and archive settings checked for validity, when validating the configuration. + // Also store the checkpoint timeout to warn in case a backup is requested without using the start-fast option. MEM_CONTEXT_BEGIN(this->pub.memContext) { this->pub.pgDataPath = strDup(varStr(varLstGet(row, 1))); this->pub.archiveMode = strDup(varStr(varLstGet(row, 2))); this->pub.archiveCommand = strDup(varStr(varLstGet(row, 3))); + this->pub.checkpointTimeout = (TimeMSec)(varUIntForce(varLstGet(row, 4))) * MSEC_PER_SEC; } MEM_CONTEXT_END(); @@ -260,6 +298,12 @@ // parallel-safe but an error will be thrown if pg_stop_backup() is run in a worker. if (dbPgVersion(this) >= PG_VERSION_PARALLEL_QUERY) dbExec(this, STRDEF("set max_parallel_workers_per_gather = 0")); + + // Is the cluster a standby? + this->pub.standby = dbIsInRecovery(this); + + // Get control file + this->pub.pgControl = pgControlFromFile(this->storage); } MEM_CONTEXT_TEMP_END(); @@ -304,12 +348,13 @@ } DbBackupStartResult -dbBackupStart(Db *this, bool startFast, bool stopAuto) +dbBackupStart(Db *const this, const bool startFast, const bool stopAuto, const bool archiveCheck) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(DB, this); FUNCTION_LOG_PARAM(BOOL, startFast); FUNCTION_LOG_PARAM(BOOL, stopAuto); + FUNCTION_LOG_PARAM(BOOL, archiveCheck); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -331,8 +376,7 @@ // If stop-auto is enabled check for a running backup if (stopAuto) { - // Feature is not supported in PostgreSQL < 9.3 - CHECK(dbPgVersion(this) >= PG_VERSION_93); + CHECK(AssertError, dbPgVersion(this) >= PG_VERSION_93, "feature not supported in PostgreSQL < " PG_VERSION_93_STR); // Feature is not needed for PostgreSQL >= 9.6 since backups are run in non-exclusive mode if (dbPgVersion(this) < PG_VERSION_96) @@ -348,14 +392,74 @@ } } + // When the start-fast option is disabled and db-timeout is smaller than checkpoint_timeout, the command may timeout + // before the backup actually starts + if (!startFast && dbDbTimeout(this) <= dbCheckpointTimeout(this)) + { + LOG_WARN_FMT( + CFGOPT_START_FAST " is disabled and " CFGOPT_DB_TIMEOUT " (%" PRIu64 "s) is smaller than the " PG_NAME + " checkpoint_timeout (%" PRIu64 "s) - timeout may occur before the backup starts", + dbDbTimeout(this) / MSEC_PER_SEC, dbCheckpointTimeout(this) / MSEC_PER_SEC); + } + + // If archive check then get the current WAL segment + const String *walSegmentCheck = NULL; + + if (archiveCheck) + { + walSegmentCheck = varStr( + dbQueryColumn( + this, + strNewFmt( + "select pg_catalog.pg_%sfile_name(pg_catalog.pg_current_%s_insert_%s())::text", + strZ(pgWalName(dbPgVersion(this))), strZ(pgWalName(dbPgVersion(this))), + strZ(pgLsnName(dbPgVersion(this)))))); + } + // Start backup VariantList *row = dbQueryRow(this, dbBackupStartQuery(dbPgVersion(this), startFast)); + // Make sure the backup start checkpoint was written to pg_control. This helps ensure that we have a consistent view of the + // storage with PostgreSQL. + const PgControl pgControl = pgControlFromFile(this->storage); + const String *const lsnStart = varStr(varLstGet(row, 0)); + + if (pgControl.checkpoint < pgLsnFromStr(lsnStart)) + { + THROW_FMT( + DbMismatchError, "current checkpoint '%s' is less than backup start '%s'", strZ(pgLsnToStr(pgControl.checkpoint)), + strZ(lsnStart)); + } + + // If archive check then make sure WAL segment was switched on start backup + const String *const walSegmentName = varStr(varLstGet(row, 1)); + + if (archiveCheck && strEq(walSegmentCheck, walSegmentName)) + { + // If the version supports restore points then force a WAL switch + if (dbPgVersion(this) >= PG_VERSION_RESTORE_POINT) + { + dbWalSwitch(this); + } + // Else disable the check. All WAL will still be checked at the end of the backup. + else + walSegmentCheck = NULL; + } + + // Check that the WAL timeline matches what is in pg_control + if (pgTimelineFromWalSegment(walSegmentName) != dbPgControl(this).timeline) + { + THROW_FMT( + DbMismatchError, "WAL timeline %u does not match " PG_FILE_PGCONTROL " timeline %u", + pgTimelineFromWalSegment(walSegmentName), dbPgControl(this).timeline); + } + // Return results MEM_CONTEXT_PRIOR_BEGIN() { - result.lsn = strDup(varStr(varLstGet(row, 0))); - result.walSegmentName = strDup(varStr(varLstGet(row, 1))); + result.lsn = strDup(lsnStart); + result.walSegmentName = strDup(walSegmentName); + result.walSegmentCheck = strDup(walSegmentCheck); } MEM_CONTEXT_PRIOR_END(); } @@ -363,6 +467,7 @@ FUNCTION_LOG_RETURN_STRUCT(result); } + /**********************************************************************************************************************************/ // Helper to build stop backup query static String * @@ -454,26 +559,6 @@ } /**********************************************************************************************************************************/ -bool -dbIsStandby(Db *this) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(DB, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - bool result = false; - - if (dbPgVersion(this) >= PG_VERSION_HOT_STANDBY) - { - result = varBool(dbQueryColumn(this, STRDEF("select pg_catalog.pg_is_in_recovery()"))); - } - - FUNCTION_LOG_RETURN(BOOL, result); -} - -/**********************************************************************************************************************************/ VariantList * dbList(Db *this) { @@ -487,20 +572,35 @@ /**********************************************************************************************************************************/ void -dbReplayWait(Db *this, const String *targetLsn, TimeMSec timeout) +dbReplayWait(Db *const this, const String *const targetLsn, const uint32_t targetTimeline, const TimeMSec timeout) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(DB, this); FUNCTION_LOG_PARAM(STRING, targetLsn); + FUNCTION_LOG_PARAM(UINT, targetTimeline); FUNCTION_LOG_PARAM(TIME_MSEC, timeout); FUNCTION_LOG_END(); ASSERT(this != NULL); ASSERT(targetLsn != NULL); + ASSERT(targetTimeline != 0); ASSERT(timeout > 0); MEM_CONTEXT_TEMP_BEGIN() { + // Check that the timeline matches the primary + if (dbPgControl(this).timeline != targetTimeline) + THROW_FMT(DbMismatchError, "standby is on timeline %u but expected %u", dbPgControl(this).timeline, targetTimeline); + + // Standby checkpoint before the backup started must be <= the target LSN. If not, it indicates that the standby was ahead + // of the primary and cannot be following it. + if (dbPgControl(this).checkpoint > pgLsnFromStr(targetLsn)) + { + THROW_FMT( + DbMismatchError, "standby checkpoint '%s' is ahead of target '%s'", strZ(pgLsnToStr(dbPgControl(this).checkpoint)), + strZ(targetLsn)); + } + // Loop until lsn has been reached or timeout Wait *wait = waitNew(timeout); bool targetReached = false; @@ -607,6 +707,54 @@ } } MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +void +dbPing(Db *const this, const bool force) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(DB, this); + FUNCTION_LOG_PARAM(BOOL, force); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Ping if forced or interval has elapsed + time_t timeNow = time(NULL); + + if (force || timeNow - this->pingTimeLast > DB_PING_SEC) + { + // Make sure recovery state has not changed for versions that support recovery + if (dbPgVersion(this) >= PG_VERSION_90) + { + if (dbIsInRecovery(this) != dbIsStandby(this)) + { + // If this is the standby then it must have been promoted + if (dbIsStandby(this)) + { + THROW( + DbMismatchError, + "standby is no longer is recovery\n" + "HINT: was the standby promoted during the backup?"); + } + // Else if a primary then something has gone seriously wrong + else + THROW(AssertError, "primary has switched to recovery"); + } + } + // Else just make sure the cluster is still running + else + dbTimeMSec(this); + + this->pingTimeLast = timeNow; + } + } + MEM_CONTEXT_TEMP_END(); FUNCTION_LOG_RETURN_VOID(); } diff -Nru pgbackrest-2.36/src/db/db.h pgbackrest-2.37/src/db/db.h --- pgbackrest-2.36/src/db/db.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/db/db.h 2022-01-03 13:43:55.000000000 +0000 @@ -9,7 +9,9 @@ #include "common/type/object.h" #include "postgres/client.h" +#include "postgres/interface.h" #include "protocol/client.h" +#include "storage/storage.h" /*********************************************************************************************************************************** Object type @@ -19,7 +21,7 @@ /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ -Db *dbNew(PgClient *client, ProtocolClient *remoteClient, const String *applicationName); +Db *dbNew(PgClient *client, ProtocolClient *remoteClient, const Storage *storage, const String *applicationName); /*********************************************************************************************************************************** Getters/Setters @@ -27,10 +29,14 @@ typedef struct DbPub { MemContext *memContext; // Mem context + PgControl pgControl; // Control info const String *archiveMode; // The archive_mode reported by the database const String *archiveCommand; // The archive_command reported by the database const String *pgDataPath; // Data directory reported by the database + bool standby; // Is the cluster a standby? unsigned int pgVersion; // Version as reported by the database + TimeMSec checkpointTimeout; // The checkpoint timeout reported by the database + TimeMSec dbTimeout; // Database timeout for statements/queries from PG client } DbPub; // Archive mode loaded from the archive_mode GUC @@ -47,6 +53,13 @@ return THIS_PUB(Db)->archiveCommand; } +// Control data +__attribute__((always_inline)) static inline PgControl +dbPgControl(const Db *const this) +{ + return THIS_PUB(Db)->pgControl; +} + // Data path loaded from the data_directory GUC __attribute__((always_inline)) static inline const String * dbPgDataPath(const Db *const this) @@ -54,6 +67,13 @@ return THIS_PUB(Db)->pgDataPath; } +// Is the cluster a standby? +__attribute__((always_inline)) static inline bool +dbIsStandby(const Db *const this) +{ + return THIS_PUB(Db)->standby; +} + // Version loaded from the server_version_num GUC __attribute__((always_inline)) static inline unsigned int dbPgVersion(const Db *const this) @@ -61,6 +81,20 @@ return THIS_PUB(Db)->pgVersion; } +// Checkpoint timeout loaded from the checkpoint_timeout GUC +__attribute__((always_inline)) static inline TimeMSec +dbCheckpointTimeout(const Db *const this) +{ + return THIS_PUB(Db)->checkpointTimeout; +} + +// Database timeout from main/remote process +__attribute__((always_inline)) static inline TimeMSec +dbDbTimeout(const Db *const this) +{ + return THIS_PUB(Db)->dbTimeout; +} + /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ @@ -72,9 +106,10 @@ { String *lsn; String *walSegmentName; + String *walSegmentCheck; // Segment used to check archiving, may be NULL } DbBackupStartResult; -DbBackupStartResult dbBackupStart(Db *this, bool startFast, bool stopAuto); +DbBackupStartResult dbBackupStart(Db *this, bool startFast, bool stopAuto, bool archiveCheck); // Stop backup and return starting lsn, wal segment name, backup label, and tablespace map typedef struct DbBackupStopResult @@ -87,14 +122,14 @@ DbBackupStopResult dbBackupStop(Db *this); -// Is this cluster a standby? -bool dbIsStandby(Db *this); - // Get list of databases in the cluster: select oid, datname, datlastsysoid from pg_database VariantList *dbList(Db *this); // Waits for replay on the standby to equal the target LSN -void dbReplayWait(Db *this, const String *targetLsn, TimeMSec timeout); +void dbReplayWait(Db *this, const String *targetLsn, uint32_t targetTimeline, TimeMSec timeout); + +// Check that the cluster is alive and correctly configured during the backup +void dbPing(Db *const this, bool force); // Epoch time on the PostgreSQL host in ms TimeMSec dbTimeMSec(Db *this); diff -Nru pgbackrest-2.36/src/db/helper.c pgbackrest-2.37/src/db/helper.c --- pgbackrest-2.36/src/db/helper.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/db/helper.c 2022-01-03 13:43:55.000000000 +0000 @@ -8,6 +8,7 @@ #include "db/helper.h" #include "postgres/interface.h" #include "protocol/helper.h" +#include "storage/helper.h" #include "version.h" /**********************************************************************************************************************************/ @@ -34,10 +35,10 @@ cfgOptionIdxStrNull(cfgOptPgSocketPath, pgIdx), cfgOptionIdxUInt(cfgOptPgPort, pgIdx), cfgOptionIdxStr(cfgOptPgDatabase, pgIdx), cfgOptionIdxStrNull(cfgOptPgUser, pgIdx), cfgOptionUInt64(cfgOptDbTimeout)), - NULL, applicationName); + NULL, storagePgIdx(pgIdx), applicationName); } else - result = dbNew(NULL, protocolRemoteGet(protocolStorageTypePg, pgIdx), applicationName); + result = dbNew(NULL, protocolRemoteGet(protocolStorageTypePg, pgIdx), storagePgIdx(pgIdx), applicationName); dbMove(result, memContextPrior()); } diff -Nru pgbackrest-2.36/src/info/infoBackup.c pgbackrest-2.37/src/info/infoBackup.c --- pgbackrest-2.36/src/info/infoBackup.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/info/infoBackup.c 2022-01-03 13:43:55.000000000 +0000 @@ -154,7 +154,7 @@ // When reading timestamps, read as uint64 to ensure always positive value (guarantee no backups before 1970) .backupTimestampStart = (time_t)varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START_VAR)), .backupTimestampStop= (time_t)varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP_VAR)), - .backupType = (BackupType)strIdFromStr(stringIdBit5, varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TYPE_VAR))), + .backupType = (BackupType)strIdFromStr(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TYPE_VAR))), // Possible NULL values .backupArchiveStart = strDup(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_ARCHIVE_START_VAR))), @@ -649,7 +649,7 @@ // If the manifest does not exist on disk and this backup has not already been deleted from the current list in the // infoBackup object, then remove it and its dependencies - if (!storageExistsP(storage, manifestFileName) && infoBackupDataByLabel(infoBackup, backupLabel) != NULL) + if (!storageExistsP(storage, manifestFileName) && infoBackupLabelExists(infoBackup, backupLabel)) { StringList *backupList = strLstSort(infoBackupDataDependentList(infoBackup, backupLabel), sortOrderDesc); @@ -695,8 +695,7 @@ // or there is a backup-prior and it is in the list, then add this backup to the current backup list if (manData->pgId == pgHistory.id && manData->pgSystemId == pgHistory.systemId && manData->pgVersion == pgHistory.version && - (manData->backupLabelPrior == NULL || - infoBackupDataByLabel(infoBackup, manData->backupLabelPrior) != NULL)) + (manData->backupLabelPrior == NULL || infoBackupLabelExists(infoBackup, manData->backupLabelPrior))) { LOG_WARN_FMT("backup '%s' found in repository added to " INFO_BACKUP_FILE, strZ(backupLabel)); infoBackupDataAdd(infoBackup, manifest); diff -Nru pgbackrest-2.36/src/info/infoBackup.h pgbackrest-2.37/src/info/infoBackup.h --- pgbackrest-2.36/src/info/infoBackup.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/info/infoBackup.h 2022-01-03 13:43:55.000000000 +0000 @@ -105,11 +105,19 @@ // Return a structure of the backup data from a specific index InfoBackupData infoBackupData(const InfoBackup *this, unsigned int backupDataIdx); +// Does the specified backup label exist? +__attribute__((always_inline)) static inline bool +infoBackupLabelExists(const InfoBackup *const this, const String *const backupLabel) +{ + ASSERT_INLINE(backupLabel != NULL); + return lstFind(THIS_PUB(InfoBackup)->backup, &backupLabel) != NULL; +} + // Return a pointer to a structure from the current backup data given a label, else NULL __attribute__((always_inline)) static inline InfoBackupData * infoBackupDataByLabel(const InfoBackup *const this, const String *const backupLabel) { - ASSERT_INLINE(backupLabel != NULL); + ASSERT_INLINE(infoBackupLabelExists(this, backupLabel)); return lstFind(THIS_PUB(InfoBackup)->backup, &backupLabel); } diff -Nru pgbackrest-2.36/src/info/info.c pgbackrest-2.37/src/info/info.c --- pgbackrest-2.36/src/info/info.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/info/info.c 2022-01-03 13:43:55.000000000 +0000 @@ -497,8 +497,7 @@ loaded = callbackFunction(callbackData, try); done = true; - // There must be at least one attempt to load the file - CHECK(loaded || try > 0); + CHECK(AssertError, loaded || try > 0, "file load must be attempted"); } CATCH_ANY() { diff -Nru pgbackrest-2.36/src/info/infoPg.c pgbackrest-2.37/src/info/infoPg.c --- pgbackrest-2.36/src/info/infoPg.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/info/infoPg.c 2022-01-03 13:43:55.000000000 +0000 @@ -185,11 +185,8 @@ this->pub.info = infoNewLoad(read, infoPgLoadCallback, &loadData); - // History must include at least one item or the file is corrupt - CHECK(!lstEmpty(this->pub.history)); - - // If the current id was not found then the file is corrupt - CHECK(loadData.currentId > 0); + CHECK(FormatError, !lstEmpty(this->pub.history), "history is missing"); + CHECK(FormatError, loadData.currentId > 0, "current id is missing"); // Find the current history item for (unsigned int historyIdx = 0; historyIdx < lstSize(this->pub.history); historyIdx++) @@ -198,8 +195,7 @@ this->historyCurrent = historyIdx; } - // If the current id did not match the history list then the file is corrupt - CHECK(this->historyCurrent != UINT_MAX); + CHECK(FormatError, this->historyCurrent != UINT_MAX, "unable to find current id in history"); } OBJ_NEW_END(); diff -Nru pgbackrest-2.36/src/info/manifest.c pgbackrest-2.37/src/info/manifest.c --- pgbackrest-2.36/src/info/manifest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/info/manifest.c 2022-01-03 13:43:55.000000000 +0000 @@ -998,8 +998,7 @@ // Else it must be a file or special (since we have already checked if it is a link) else { - // Tablespace links should never be to a file - CHECK(target.tablespaceId == 0); + CHECK(FormatError, target.tablespaceId == 0, "tablespace links to a file"); // Identify target as a file target.path = strPath(info->linkDestination); @@ -1548,7 +1547,7 @@ // If bool then it should be false. This indicates that the owner could not be mapped to a name during the backup. if (varType(owner) == varTypeBool) { - CHECK(!varBool(owner)); + CHECK(FormatError, !varBool(owner), "owner bool must be false"); FUNCTION_TEST_RETURN(NULL); } @@ -1569,7 +1568,7 @@ if (varType(ownerDefault) == varTypeBool) { // Value must be false - CHECK(!varBool(ownerDefault)); + CHECK(FormatError, !varBool(ownerDefault), "owner bool must be false"); FUNCTION_TEST_RETURN(BOOL_FALSE_VAR); } @@ -1865,7 +1864,7 @@ manifest->pub.data.backupTimestampStop = (time_t)varUInt64(value); else if (strEq(key, MANIFEST_KEY_BACKUP_TYPE_STR)) { - manifest->pub.data.backupType = (BackupType)strIdFromStr(stringIdBit5, varStr(value)); + manifest->pub.data.backupType = (BackupType)strIdFromStr(varStr(value)); ASSERT( manifest->pub.data.backupType == backupTypeFull || manifest->pub.data.backupType == backupTypeDiff || manifest->pub.data.backupType == backupTypeIncr); @@ -1903,7 +1902,7 @@ // This new option allows any type of compression to be specified. It must be parsed after the option above so the // value does not get overwritten. Since options are stored in alpha order this should always be true. else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_TYPE_STR)) - manifest->pub.data.backupOptionCompressType = compressTypeEnum(varStr(value)); + manifest->pub.data.backupOptionCompressType = compressTypeEnum(strIdFromStr(varStr(value))); else if (strEq(key, MANIFEST_KEY_OPTION_HARDLINK_STR)) manifest->pub.data.backupOptionHardLink = varBool(value); else if (strEq(key, MANIFEST_KEY_OPTION_ONLINE_STR)) diff -Nru pgbackrest-2.36/src/main.c pgbackrest-2.37/src/main.c --- pgbackrest-2.36/src/main.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/main.c 2022-01-03 13:43:55.000000000 +0000 @@ -233,10 +233,10 @@ cmdStorageRemove(); break; - // Server start command + // Server command // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdServerStart: - cmdServer(UINT64_MAX); + case cfgCmdServer: + cmdServer((unsigned int)argListSize, argList); break; // Server ping command diff -Nru pgbackrest-2.36/src/Makefile.in pgbackrest-2.37/src/Makefile.in --- pgbackrest-2.36/src/Makefile.in 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/Makefile.in 2022-01-03 13:43:55.000000000 +0000 @@ -39,6 +39,7 @@ common/type/xml.c \ common/user.c \ common/wait.c \ + config/common.c \ storage/posix/read.c \ storage/posix/storage.c \ storage/posix/write.c \ diff -Nru pgbackrest-2.36/src/postgres/client.c pgbackrest-2.37/src/postgres/client.c --- pgbackrest-2.36/src/postgres/client.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/client.c 2022-01-03 13:43:55.000000000 +0000 @@ -16,13 +16,8 @@ ***********************************************************************************************************************************/ struct PgClient { - const String *host; - unsigned int port; - const String *database; - const String *user; - TimeMSec queryTimeout; - - PGconn *connection; + PgClientPub pub; // Publicly accessible variables + PGconn *connection; // Pg connection }; /*********************************************************************************************************************************** @@ -46,14 +41,14 @@ /**********************************************************************************************************************************/ PgClient * -pgClientNew(const String *host, const unsigned int port, const String *database, const String *user, const TimeMSec queryTimeout) +pgClientNew(const String *host, const unsigned int port, const String *database, const String *user, const TimeMSec timeout) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, host); FUNCTION_LOG_PARAM(UINT, port); FUNCTION_LOG_PARAM(STRING, database); FUNCTION_LOG_PARAM(STRING, user); - FUNCTION_LOG_PARAM(TIME_MSEC, queryTimeout); + FUNCTION_LOG_PARAM(TIME_MSEC, timeout); FUNCTION_LOG_END(); ASSERT(port >= 1 && port <= 65535); @@ -67,11 +62,14 @@ *this = (PgClient) { - .host = strDup(host), - .port = port, - .database = strDup(database), - .user = strDup(user), - .queryTimeout = queryTimeout, + .pub = + { + .host = strDup(host), + .port = port, + .database = strDup(database), + .user = strDup(user), + .timeout = timeout, + }, }; } OBJ_NEW_END(); @@ -129,20 +127,21 @@ FUNCTION_LOG_END(); ASSERT(this != NULL); - CHECK(this->connection == NULL); + CHECK(AssertError, this->connection == NULL, "invalid connection"); MEM_CONTEXT_TEMP_BEGIN() { // Base connection string - String *connInfo = strCatFmt(strNew(), "dbname=%s port=%u", strZ(pgClientEscape(this->database)), this->port); + String *connInfo = strCatFmt( + strNew(), "dbname=%s port=%u", strZ(pgClientEscape(pgClientDatabase(this))), pgClientPort(this)); // Add user if specified - if (this->user != NULL) - strCatFmt(connInfo, " user=%s", strZ(pgClientEscape(this->user))); + if (pgClientUser(this) != NULL) + strCatFmt(connInfo, " user=%s", strZ(pgClientEscape(pgClientUser(this)))); // Add host if specified - if (this->host != NULL) - strCatFmt(connInfo, " host=%s", strZ(pgClientEscape(this->host))); + if (pgClientHost(this) != NULL) + strCatFmt(connInfo, " host=%s", strZ(pgClientEscape(pgClientHost(this)))); // Make the connection this->connection = PQconnectdb(strZ(connInfo)); @@ -176,7 +175,7 @@ FUNCTION_LOG_END(); ASSERT(this != NULL); - CHECK(this->connection != NULL); + CHECK(AssertError, this->connection != NULL, "invalid connection"); ASSERT(query != NULL); VariantList *result = NULL; @@ -192,7 +191,7 @@ } // Wait for a result - Wait *wait = waitNew(this->queryTimeout); + Wait *wait = waitNew(pgClientTimeout(this)); bool busy = false; do @@ -232,7 +231,7 @@ { // Throw timeout error if cancelled if (busy) - THROW_FMT(DbQueryError, "query '%s' timed out after %" PRIu64 "ms", strZ(query), this->queryTimeout); + THROW_FMT(DbQueryError, "query '%s' timed out after %" PRIu64 "ms", strZ(query), pgClientTimeout(this)); // If this was a command that returned no results then we are done ExecStatusType resultStatus = PQresultStatus(pgResult); @@ -320,8 +319,7 @@ // Free the result PQclear(pgResult); - // Need to get a NULL result to complete the request - CHECK(PQgetResult(this->connection) == NULL); + CHECK(ServiceError, PQgetResult(this->connection) == NULL, "NULL result required to complete request"); } TRY_END(); @@ -357,6 +355,6 @@ pgClientToLog(const PgClient *this) { return strNewFmt( - "{host: %s, port: %u, database: %s, user: %s, queryTimeout %" PRIu64 "}", strZ(strToLog(this->host)), this->port, - strZ(strToLog(this->database)), strZ(strToLog(this->user)), this->queryTimeout); + "{host: %s, port: %u, database: %s, user: %s, queryTimeout %" PRIu64 "}", strZ(strToLog(pgClientHost(this))), + pgClientPort(this), strZ(strToLog(pgClientDatabase(this))), strZ(strToLog(pgClientUser(this))), pgClientTimeout(this)); } diff -Nru pgbackrest-2.36/src/postgres/client.h pgbackrest-2.37/src/postgres/client.h --- pgbackrest-2.36/src/postgres/client.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/client.h 2022-01-03 13:43:55.000000000 +0000 @@ -22,7 +22,54 @@ Constructors ***********************************************************************************************************************************/ PgClient *pgClientNew( - const String *host, const unsigned int port, const String *database, const String *user, const TimeMSec queryTimeout); + const String *host, const unsigned int port, const String *database, const String *user, const TimeMSec timeout); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +typedef struct PgClientPub +{ + const String *host; // Pg host + unsigned int port; // Pg port + const String *database; // Pg database + const String *user; // Pg user + TimeMSec timeout; // Timeout for statements/queries +} PgClientPub; + +// Pg host +__attribute__((always_inline)) static inline const String * +pgClientHost(const PgClient *const this) +{ + return THIS_PUB(PgClient)->host; +} + +// Pg port +__attribute__((always_inline)) static inline unsigned int +pgClientPort(const PgClient *const this) +{ + return THIS_PUB(PgClient)->port; +} + +// Pg database +__attribute__((always_inline)) static inline const String * +pgClientDatabase(const PgClient *const this) +{ + return THIS_PUB(PgClient)->database; +} + +// Pg user +__attribute__((always_inline)) static inline const String * +pgClientUser(const PgClient *const this) +{ + return THIS_PUB(PgClient)->user; +} + +// Timeout for statements/queries +__attribute__((always_inline)) static inline TimeMSec +pgClientTimeout(const PgClient *const this) +{ + return THIS_PUB(PgClient)->timeout; +} /*********************************************************************************************************************************** Functions diff -Nru pgbackrest-2.36/src/postgres/interface/pageChecksum.vendor.c pgbackrest-2.37/src/postgres/interface/pageChecksum.vendor.c --- pgbackrest-2.36/src/postgres/interface/pageChecksum.vendor.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/interface/pageChecksum.vendor.c 2022-01-03 13:43:55.000000000 +0000 @@ -24,7 +24,7 @@ * referenced by storage/checksum.h. (Note: you may need to redefine * Assert() as empty to compile this successfully externally.) * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * src/include/storage/checksum_impl.h diff -Nru pgbackrest-2.36/src/postgres/interface/static.vendor.h pgbackrest-2.37/src/postgres/interface/static.vendor.h --- pgbackrest-2.36/src/postgres/interface/static.vendor.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/interface/static.vendor.h 2022-01-03 13:43:55.000000000 +0000 @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** PostgreSQL Types That Do Not Vary By Version -Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group +Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group Portions Copyright (c) 1994, Regents of the University of California For each supported release of PostgreSQL check the types in this file to see if they have changed. The easiest way to do this is to diff -Nru pgbackrest-2.36/src/postgres/interface/version.intern.h pgbackrest-2.37/src/postgres/interface/version.intern.h --- pgbackrest-2.36/src/postgres/interface/version.intern.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/interface/version.intern.h 2022-01-03 13:43:55.000000000 +0000 @@ -73,6 +73,8 @@ { \ .systemId = ((ControlFileData *)controlFile)->system_identifier, \ .catalogVersion = ((ControlFileData *)controlFile)->catalog_version_no, \ + .checkpoint = ((ControlFileData *)controlFile)->checkPoint, \ + .timeline = ((ControlFileData *)controlFile)->checkPointCopy.ThisTimeLineID, \ .pageSize = ((ControlFileData *)controlFile)->blcksz, \ .walSegmentSize = ((ControlFileData *)controlFile)->xlog_seg_size, \ .pageChecksum = ((ControlFileData *)controlFile)->data_checksum_version != 0, \ @@ -92,6 +94,10 @@ { \ .systemId = ((ControlFileData *)controlFile)->system_identifier, \ .catalogVersion = ((ControlFileData *)controlFile)->catalog_version_no, \ + .checkpoint = \ + (uint64_t)((ControlFileData *)controlFile)->checkPoint.xlogid << 32 | \ + ((ControlFileData *)controlFile)->checkPoint.xrecoff, \ + .timeline = ((ControlFileData *)controlFile)->checkPointCopy.ThisTimeLineID, \ .pageSize = ((ControlFileData *)controlFile)->blcksz, \ .walSegmentSize = ((ControlFileData *)controlFile)->xlog_seg_size, \ }; \ diff -Nru pgbackrest-2.36/src/postgres/interface/version.vendor.h pgbackrest-2.37/src/postgres/interface/version.vendor.h --- pgbackrest-2.36/src/postgres/interface/version.vendor.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/interface/version.vendor.h 2022-01-03 13:43:55.000000000 +0000 @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** PostgreSQL Types That Vary By Version -Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group +Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group Portions Copyright (c) 1994, Regents of the University of California For each supported release of PostgreSQL check the types in this file to see if they have changed. The easiest way to do this is to diff -Nru pgbackrest-2.36/src/postgres/interface.c pgbackrest-2.37/src/postgres/interface.c --- pgbackrest-2.36/src/postgres/interface.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/interface.c 2022-01-03 13:43:55.000000000 +0000 @@ -490,7 +490,7 @@ { StringList *lsnPart = strLstNewSplit(lsn, FSLASH_STR); - CHECK(strLstSize(lsnPart) == 2); + CHECK(FormatError, strLstSize(lsnPart) == 2, "lsn requires two parts"); result = (cvtZToUInt64Base(strZ(strLstGet(lsnPart, 0)), 16) << 32) + cvtZToUInt64Base(strZ(strLstGet(lsnPart, 1)), 16); } @@ -541,6 +541,25 @@ } /**********************************************************************************************************************************/ +uint32_t +pgTimelineFromWalSegment(const String *const walSegment) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, walSegment); + FUNCTION_TEST_END(); + + ASSERT(walSegment != NULL); + ASSERT(strSize(walSegment) == 24); + + char buffer[9]; + + strncpy(buffer, strZ(walSegment), sizeof(buffer) - 1); + buffer[sizeof(buffer) - 1] = '\0'; + + FUNCTION_TEST_RETURN(cvtZToUIntBase(buffer, 16)); +} + +/**********************************************************************************************************************************/ StringList * pgLsnRangeToWalSegmentList( unsigned int pgVersion, uint32_t timeline, uint64_t lsnStart, uint64_t lsnStop, unsigned int walSegmentSize) diff -Nru pgbackrest-2.36/src/postgres/interface.h pgbackrest-2.37/src/postgres/interface.h --- pgbackrest-2.36/src/postgres/interface.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/postgres/interface.h 2022-01-03 13:43:55.000000000 +0000 @@ -105,6 +105,9 @@ uint64_t systemId; unsigned int catalogVersion; + uint64_t checkpoint; // Last checkpoint LSN + uint32_t timeline; // Current timeline + unsigned int pageSize; unsigned int walSegmentSize; @@ -149,6 +152,9 @@ String *pgLsnToWalSegment(uint32_t timeline, uint64_t lsn, unsigned int walSegmentSize); uint64_t pgLsnFromWalSegment(const String *walSegment, unsigned int walSegmentSize); +// Get timeline from WAL segment name +uint32_t pgTimelineFromWalSegment(const String *walSegment); + // Convert a timeline and lsn range to a list of wal segments StringList *pgLsnRangeToWalSegmentList( unsigned int pgVersion, uint32_t timeline, uint64_t lsnStart, uint64_t lsnStop, unsigned int walSegmentSize); diff -Nru pgbackrest-2.36/src/protocol/client.c pgbackrest-2.37/src/protocol/client.c --- pgbackrest-2.36/src/protocol/client.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/protocol/client.c 2022-01-03 13:43:55.000000000 +0000 @@ -243,7 +243,7 @@ // Switch state to idle after error (server will do the same) this->state = protocolClientStateIdle; - CHECK(message != NULL && stack != NULL); + CHECK(FormatError, message != NULL && stack != NULL, "invalid error data"); errorInternalThrow(type, __FILE__, __func__, __LINE__, strZ(message), strZ(stack)); } @@ -276,7 +276,7 @@ protocolClientError(this, type, response); - CHECK(type == protocolMessageTypeData); + CHECK(FormatError, type == protocolMessageTypeData, "expected data message"); MEM_CONTEXT_PRIOR_BEGIN() { @@ -315,7 +315,7 @@ protocolClientError(this, type, response); - CHECK(type == protocolMessageTypeDataEnd); + CHECK(FormatError, type == protocolMessageTypeDataEnd, "expected data end message"); pckReadEndP(response); diff -Nru pgbackrest-2.36/src/protocol/helper.c pgbackrest-2.37/src/protocol/helper.c --- pgbackrest-2.36/src/protocol/helper.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/protocol/helper.c 2022-01-03 13:43:55.000000000 +0000 @@ -163,7 +163,7 @@ // Only enable file logging on the local when requested kvPut( optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_FILE), - cfgOptionBool(cfgOptLogSubprocess) ? VARSTR(cfgOptionStr(cfgOptLogLevelFile)) : VARSTRDEF("off")); + cfgOptionBool(cfgOptLogSubprocess) ? VARUINT64(cfgOptionStrId(cfgOptLogLevelFile)) : VARSTRDEF("off")); // Always output errors on stderr for debugging purposes kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_STDERR), VARSTRDEF("error")); @@ -382,7 +382,8 @@ TRY_BEGIN() { // Get list of authorized stanzas for this client - CHECK(cfgOptionTest(cfgOptTlsServerAuth)); + CHECK(AssertError, cfgOptionTest(cfgOptTlsServerAuth), "missing auth data"); + const String *const clientAuthList = strDup( varStr(kvGet(cfgOptionKv(cfgOptTlsServerAuth), VARSTR(ioSessionPeerName(tlsSession))))); @@ -392,7 +393,7 @@ // Get parameter list from the client and load it const ProtocolServerCommandGetResult command = protocolServerCommandGet(result); - CHECK(command.id == PROTOCOL_COMMAND_CONFIG); + CHECK(FormatError, command.id == PROTOCOL_COMMAND_CONFIG, "expected config command"); StringList *const paramList = pckReadStrLstP(pckReadNew(command.param)); strLstInsert(paramList, 0, cfgExe()); @@ -551,7 +552,7 @@ // Only enable file logging on the remote when requested kvPut( optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_FILE), - cfgOptionBool(cfgOptLogSubprocess) ? VARSTR(cfgOptionStr(cfgOptLogLevelFile)) : VARSTRDEF("off")); + cfgOptionBool(cfgOptLogSubprocess) ? VARUINT64(cfgOptionStrId(cfgOptLogLevelFile)) : VARSTRDEF("off")); // Always output errors on stderr for debugging purposes kvPut(optionReplace, VARSTRDEF(CFGOPT_LOG_LEVEL_STDERR), VARSTRDEF("error")); @@ -669,15 +670,15 @@ ASSERT(remoteType == CFGOPTVAL_REPO_HOST_TYPE_TLS); // Negotiate TLS - helper->ioClient = tlsClientNew( + helper->ioClient = tlsClientNewP( sckClientNew( host, cfgOptionIdxUInt(isRepo ? cfgOptRepoHostPort : cfgOptPgHostPort, hostIdx), cfgOptionUInt64(cfgOptIoTimeout), cfgOptionUInt64(cfgOptProtocolTimeout)), host, cfgOptionUInt64(cfgOptIoTimeout), cfgOptionUInt64(cfgOptProtocolTimeout), true, - cfgOptionIdxStrNull(isRepo ? cfgOptRepoHostCaFile : cfgOptPgHostCaFile, hostIdx), - cfgOptionIdxStrNull(isRepo ? cfgOptRepoHostCaPath : cfgOptPgHostCaPath, hostIdx), - cfgOptionIdxStr(isRepo ? cfgOptRepoHostCertFile : cfgOptPgHostCertFile, hostIdx), - cfgOptionIdxStr(isRepo ? cfgOptRepoHostKeyFile : cfgOptPgHostKeyFile, hostIdx)); + .caFile = cfgOptionIdxStrNull(isRepo ? cfgOptRepoHostCaFile : cfgOptPgHostCaFile, hostIdx), + .caPath = cfgOptionIdxStrNull(isRepo ? cfgOptRepoHostCaPath : cfgOptPgHostCaPath, hostIdx), + .certFile = cfgOptionIdxStr(isRepo ? cfgOptRepoHostCertFile : cfgOptPgHostCertFile, hostIdx), + .keyFile = cfgOptionIdxStr(isRepo ? cfgOptRepoHostKeyFile : cfgOptPgHostKeyFile, hostIdx)); helper->ioSession = ioClientOpen(helper->ioClient); read = ioSessionIoRead(helper->ioSession); @@ -753,7 +754,7 @@ if (cfgOptionTest(cfgOptProcess)) processId = cfgOptionUInt(cfgOptProcess); - CHECK(hostIdx < protocolHelper.clientRemoteSize); + CHECK(AssertError, hostIdx < protocolHelper.clientRemoteSize, "invalid host"); // Create protocol object ProtocolHelperClient *protocolHelperClient = &protocolHelper.clientRemote[hostIdx]; @@ -777,7 +778,7 @@ VariantList *optionList = configOptionRemote(protocolHelperClient->client, param); - if (!strEq(varStr(varLstGet(optionList, 0)), strIdToStr(cipherTypeNone))) + if (varUInt64(varLstGet(optionList, 0)) != cipherTypeNone) { cfgOptionIdxSet(cfgOptRepoCipherType, hostIdx, cfgSourceConfig, varLstGet(optionList, 0)); cfgOptionIdxSet(cfgOptRepoCipherPass, hostIdx, cfgSourceConfig, varLstGet(optionList, 1)); diff -Nru pgbackrest-2.36/src/protocol/server.c pgbackrest-2.37/src/protocol/server.c --- pgbackrest-2.36/src/protocol/server.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/protocol/server.c 2022-01-03 13:43:55.000000000 +0000 @@ -119,7 +119,7 @@ PackRead *const command = pckReadNewIo(this->read); ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(command); - CHECK(type == protocolMessageTypeCommand); + CHECK(FormatError, type == protocolMessageTypeCommand, "expected command message"); MEM_CONTEXT_PRIOR_BEGIN() { @@ -314,7 +314,7 @@ PackRead *data = pckReadNewIo(this->read); ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(data); - CHECK(type == protocolMessageTypeData); + CHECK(FormatError, type == protocolMessageTypeData, "expected data message"); MEM_CONTEXT_PRIOR_BEGIN() { diff -Nru pgbackrest-2.36/src/storage/azure/helper.c pgbackrest-2.37/src/storage/azure/helper.c --- pgbackrest-2.36/src/storage/azure/helper.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/azure/helper.c 2022-01-03 13:43:55.000000000 +0000 @@ -4,6 +4,7 @@ #include "build.auto.h" #include "common/debug.h" +#include "common/io/http/url.h" #include "common/io/io.h" #include "common/log.h" #include "config/config.h" @@ -21,27 +22,37 @@ ASSERT(cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == STORAGE_AZURE_TYPE); - const String *endpoint = cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx); - const String *const host = cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx); + // Parse the endpoint url + const HttpUrl *const url = httpUrlNewParseP(cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx), .type = httpProtocolTypeHttps); + const String *endpoint = httpUrlHost(url); + unsigned int port = httpUrlPort(url); + StorageAzureUriStyle uriStyle = (StorageAzureUriStyle)cfgOptionIdxStrId(cfgOptRepoAzureUriStyle, repoIdx); // If the host is set then set it as the endpoint. The host option is used to set path-style URIs when working with Azurite. // This was ill-advised, so the uri-style option was added to allow the user to select the URI style used by the server. // Preserve the old behavior when uri-style is defaulted. - if (host != NULL) + if (cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx) != NULL) { - endpoint = host; + const HttpUrl *const url = httpUrlNewParseP(cfgOptionIdxStr(cfgOptRepoStorageHost, repoIdx), .type = httpProtocolTypeHttps); + + endpoint = httpUrlHost(url); + port = httpUrlPort(url); if (cfgOptionIdxSource(cfgOptRepoAzureUriStyle, repoIdx) == cfgSourceDefault) uriStyle = storageAzureUriStylePath; } + // If port was specified, overwrite the parsed/default port + if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault) + port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx); + Storage *const result = storageAzureNew( cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), (StorageAzureKeyType)cfgOptionIdxStrId(cfgOptRepoAzureKeyType, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureKey, repoIdx), STORAGE_AZURE_BLOCKSIZE_MIN, endpoint, uriStyle, - cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), + port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); FUNCTION_LOG_RETURN(STORAGE, result); diff -Nru pgbackrest-2.36/src/storage/azure/storage.c pgbackrest-2.37/src/storage/azure/storage.c --- pgbackrest-2.36/src/storage/azure/storage.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/azure/storage.c 2022-01-03 13:43:55.000000000 +0000 @@ -737,9 +737,9 @@ // Create the http client used to service requests driver->httpClient = httpClientNew( - tlsClientNew( - sckClientNew(driver->host, port, timeout, timeout), driver->host, timeout, timeout, verifyPeer, caFile, caPath, - NULL, NULL), + tlsClientNewP( + sckClientNew(driver->host, port, timeout, timeout), driver->host, timeout, timeout, verifyPeer, .caFile = caFile, + .caPath = caPath), timeout); // Create list of redacted headers diff -Nru pgbackrest-2.36/src/storage/gcs/storage.c pgbackrest-2.37/src/storage/gcs/storage.c --- pgbackrest-2.36/src/storage/gcs/storage.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/gcs/storage.c 2022-01-03 13:43:55.000000000 +0000 @@ -138,13 +138,13 @@ { // Get token result.tokenType = strDup(varStr(kvGet(kvResponse, GCS_JSON_TOKEN_TYPE_VAR))); - CHECK(result.tokenType != NULL); + CHECK(FormatError, result.tokenType != NULL, "token type missing"); result.token = strDup(varStr(kvGet(kvResponse, GCS_JSON_ACCESS_TOKEN_VAR))); - CHECK(result.token != NULL); + CHECK(FormatError, result.token != NULL, "access token missing"); // Get expiration const Variant *const expiresIn = kvGet(kvResponse, GCS_JSON_EXPIRES_IN_VAR); - CHECK(expiresIn != NULL); + CHECK(FormatError, expiresIn != NULL, "expiry missing"); result.timeExpire = timeBegin + (time_t)varInt64Force(expiresIn); } @@ -648,7 +648,7 @@ for (unsigned int fileIdx = 0; fileIdx < varLstSize(fileList); fileIdx++) { const KeyValue *file = varKv(varLstGet(fileList, fileIdx)); - CHECK(file != NULL); + CHECK(FormatError, file != NULL, "file missing"); // Get file name StorageInfo info = @@ -658,7 +658,7 @@ .exists = true, }; - CHECK(info.name != NULL); + CHECK(FormatError, info.name != NULL, "file name missing"); // Strip off the base prefix when present if (!strEmpty(basePrefix)) @@ -968,14 +968,14 @@ driver->privateKey = varStr(kvGet(kvKey, GCS_JSON_PRIVATE_KEY_VAR)); const String *const uri = varStr(kvGet(kvKey, GCS_JSON_TOKEN_URI_VAR)); - CHECK(driver->credential != NULL && driver->privateKey != NULL && uri != NULL); + CHECK(FormatError, driver->credential != NULL && driver->privateKey != NULL && uri != NULL, "credentials missing"); driver->authUrl = httpUrlNewParseP(uri, .type = httpProtocolTypeHttps); driver->authClient = httpClientNew( - tlsClientNew( + tlsClientNewP( sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout, timeout), - httpUrlHost(driver->authUrl), timeout, timeout, verifyPeer, caFile, caPath, NULL, NULL), + httpUrlHost(driver->authUrl), timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath), timeout); break; @@ -993,9 +993,9 @@ // Create the http client used to service requests driver->httpClient = httpClientNew( - tlsClientNew( + tlsClientNewP( sckClientNew(driver->endpoint, httpUrlPort(url), timeout, timeout), driver->endpoint, timeout, timeout, verifyPeer, - caFile, caPath, NULL, NULL), + .caFile = caFile, .caPath = caPath), timeout); // Create list of redacted headers diff -Nru pgbackrest-2.36/src/storage/gcs/write.c pgbackrest-2.37/src/storage/gcs/write.c --- pgbackrest-2.36/src/storage/gcs/write.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/gcs/write.c 2022-01-03 13:43:55.000000000 +0000 @@ -85,7 +85,7 @@ // Check the md5 hash const String *md5base64 = varStr(kvGet(content, GCS_JSON_MD5_HASH_VAR)); - CHECK(md5base64 != NULL); + CHECK(FormatError, md5base64 != NULL, "MD5 missing"); const String *md5actual = bufHex(bufNewDecode(encodeBase64, md5base64)); const String *md5expected = pckReadStrP(pckReadNew(ioFilterResult(this->md5hash))); @@ -175,7 +175,7 @@ MEM_CONTEXT_BEGIN(THIS_MEM_CONTEXT()) { this->uploadId = strDup(httpHeaderGet(httpResponseHeader(response), GCS_HEADER_UPLOAD_ID_STR)); - CHECK(this->uploadId != NULL); + CHECK(FormatError, this->uploadId != NULL, "upload id missing"); } MEM_CONTEXT_END(); } diff -Nru pgbackrest-2.36/src/storage/helper.c pgbackrest-2.37/src/storage/helper.c --- pgbackrest-2.36/src/storage/helper.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/helper.c 2022-01-03 13:43:55.000000000 +0000 @@ -381,7 +381,7 @@ // If no helper was found it try Posix if (result == NULL) { - CHECK(type == STORAGE_POSIX_TYPE); + CHECK(AssertError, type == STORAGE_POSIX_TYPE, "invalid storage type"); result = storagePosixNewP( cfgOptionIdxStr(cfgOptRepoPath, repoIdx), .write = write, .pathExpressionFunction = storageRepoPathExpression); diff -Nru pgbackrest-2.36/src/storage/s3/helper.c pgbackrest-2.37/src/storage/s3/helper.c --- pgbackrest-2.36/src/storage/s3/helper.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/s3/helper.c 2022-01-03 13:43:55.000000000 +0000 @@ -6,6 +6,7 @@ #include #include "common/debug.h" +#include "common/io/http/url.h" #include "common/io/io.h" #include "common/log.h" #include "config/config.h" @@ -24,14 +25,23 @@ ASSERT(cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == STORAGE_S3_TYPE); - // Set the default port - unsigned int port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx); + // Parse the endpoint url + const HttpUrl *const url = httpUrlNewParseP(cfgOptionIdxStr(cfgOptRepoS3Endpoint, repoIdx), .type = httpProtocolTypeHttps); + const String *const endPoint = httpUrlHost(url); + unsigned int port = httpUrlPort(url); + + // If host was specified then use it + const String *host = NULL; + + if (cfgOptionIdxSource(cfgOptRepoStorageHost, repoIdx) != cfgSourceDefault) + { + const HttpUrl *const url = httpUrlNewParseP(cfgOptionIdxStr(cfgOptRepoStorageHost, repoIdx), .type = httpProtocolTypeHttps); + + host = httpUrlHost(url); + port = httpUrlPort(url); + } - // Extract port from the endpoint and host if it is present - const String *const endPoint = cfgOptionIdxHostPort(cfgOptRepoS3Endpoint, repoIdx, &port); - const String *const host = cfgOptionIdxHostPort(cfgOptRepoStorageHost, repoIdx, &port); - - // If the port option was set explicitly then use it in preference to appended ports + // If port was specified, overwrite the parsed/default port if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault) port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx); diff -Nru pgbackrest-2.36/src/storage/s3/storage.c pgbackrest-2.37/src/storage/s3/storage.c --- pgbackrest-2.36/src/storage/s3/storage.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/s3/storage.c 2022-01-03 13:43:55.000000000 +0000 @@ -331,15 +331,15 @@ { // Check the code field for errors const Variant *code = kvGetDefault(credential, S3_JSON_TAG_CODE_VAR, VARSTRDEF("code field is missing")); - CHECK(code != NULL); + CHECK(FormatError, code != NULL, "error code missing"); if (!varEq(code, S3_JSON_VALUE_SUCCESS_VAR)) THROW_FMT(FormatError, "unable to retrieve temporary credentials: %s", strZ(varStr(code))); // Make sure the required values are present - CHECK(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR) != NULL); - CHECK(kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR) != NULL); - CHECK(kvGet(credential, S3_JSON_TAG_TOKEN_VAR) != NULL); + CHECK(FormatError, kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR) != NULL, "access key missing"); + CHECK(FormatError, kvGet(credential, S3_JSON_TAG_SECRET_ACCESS_KEY_VAR) != NULL, "secret access key missing"); + CHECK(FormatError, kvGet(credential, S3_JSON_TAG_TOKEN_VAR) != NULL, "token missing"); // Copy credentials this->accessKey = strDup(varStr(kvGet(credential, S3_JSON_TAG_ACCESS_KEY_ID_VAR))); @@ -349,7 +349,7 @@ MEM_CONTEXT_END(); // Update expiration time - CHECK(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR) != NULL); + CHECK(FormatError, kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR) != NULL, "expiration missing"); this->credExpirationTime = storageS3CvtTime(varStr(kvGet(credential, S3_JSON_TAG_EXPIRATION_VAR))); FUNCTION_LOG_RETURN_VOID(); @@ -383,7 +383,7 @@ this->credHttpClient, HTTP_VERB_GET_STR, FSLASH_STR, .header = header, .query = query); HttpResponse *const response = httpRequestResponse(request, true); - CHECK(httpResponseCode(response) != HTTP_RESPONSE_CODE_NOT_FOUND); + CHECK(FormatError, httpResponseCode(response) != HTTP_RESPONSE_CODE_NOT_FOUND, "invalid response code"); // Copy credentials const XmlNode *const xmlCred = @@ -754,11 +754,11 @@ const HttpHeader *httpHeader = httpResponseHeader(httpResponse); const String *const contentLength = httpHeaderGet(httpHeader, HTTP_HEADER_CONTENT_LENGTH_STR); - CHECK(contentLength != NULL); + CHECK(FormatError, contentLength != NULL, "content length missing"); result.size = cvtZToUInt64(strZ(contentLength)); const String *const lastModified = httpHeaderGet(httpHeader, HTTP_HEADER_LAST_MODIFIED_STR); - CHECK(lastModified != NULL); + CHECK(FormatError, lastModified != NULL, "last modified missing"); result.timeModified = httpDateToTime(lastModified); } @@ -1077,8 +1077,8 @@ host = driver->bucketEndpoint; driver->httpClient = httpClientNew( - tlsClientNew( - sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, caFile, caPath, NULL, NULL), + tlsClientNewP( + sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath), timeout); // Initialize authentication @@ -1110,9 +1110,9 @@ driver->credHost = S3_STS_HOST_STR; driver->credExpirationTime = time(NULL); driver->credHttpClient = httpClientNew( - tlsClientNew( + tlsClientNewP( sckClientNew(driver->credHost, S3_STS_PORT, timeout, timeout), driver->credHost, timeout, timeout, true, - caFile, caPath, NULL, NULL), + .caFile = caFile, .caPath = caPath), timeout); break; diff -Nru pgbackrest-2.36/src/storage/storage.c pgbackrest-2.37/src/storage/storage.c --- pgbackrest-2.36/src/storage/storage.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/storage/storage.c 2022-01-03 13:43:55.000000000 +0000 @@ -75,13 +75,19 @@ }; // If path sync feature is enabled then path feature must be enabled - CHECK(!storageFeature(this, storageFeaturePathSync) || storageFeature(this, storageFeaturePath)); + CHECK( + AssertError, !storageFeature(this, storageFeaturePathSync) || storageFeature(this, storageFeaturePath), + "path feature required"); // If hardlink feature is enabled then path feature must be enabled - CHECK(!storageFeature(this, storageFeatureHardLink) || storageFeature(this, storageFeaturePath)); + CHECK( + AssertError, !storageFeature(this, storageFeatureHardLink) || storageFeature(this, storageFeaturePath), + "path feature required"); // If symlink feature is enabled then path feature must be enabled - CHECK(!storageFeature(this, storageFeatureSymLink) || storageFeature(this, storageFeaturePath)); + CHECK( + AssertError, !storageFeature(this, storageFeatureSymLink) || storageFeature(this, storageFeaturePath), + "path feature required"); FUNCTION_LOG_RETURN(STORAGE, this); } diff -Nru pgbackrest-2.36/src/version.h pgbackrest-2.37/src/version.h --- pgbackrest-2.36/src/version.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/src/version.h 2022-01-03 13:43:55.000000000 +0000 @@ -33,6 +33,6 @@ /*********************************************************************************************************************************** Software version ***********************************************************************************************************************************/ -#define PROJECT_VERSION "2.36" +#define PROJECT_VERSION "2.37" #endif diff -Nru pgbackrest-2.36/test/code-count/file-type.yaml pgbackrest-2.37/test/code-count/file-type.yaml --- pgbackrest-2.36/test/code-count/file-type.yaml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/code-count/file-type.yaml 2022-01-03 13:43:55.000000000 +0000 @@ -1267,6 +1267,14 @@ class: core type: c/h +src/config/common.c: + class: core + type: c + +src/config/common.h: + class: core + type: c/h + src/config/config.auto.h: class: core/auto type: c/h diff -Nru pgbackrest-2.36/test/define.yaml pgbackrest-2.37/test/define.yaml --- pgbackrest-2.36/test/define.yaml 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/define.yaml 2022-01-03 13:43:55.000000000 +0000 @@ -96,6 +96,7 @@ # ---------------------------------------------------------------------------------------------------------------------------- - name: assert-on + feature: assert total: 1 coverage: @@ -386,6 +387,7 @@ depend: - command/backup/pageChecksum - common/lock + - config/common - config/config - config/parse - config/exec @@ -426,27 +428,6 @@ - common/lock # ******************************************************************************************************************************** - - name: postgres - - test: - # ---------------------------------------------------------------------------------------------------------------------------- - - name: client - total: 1 - harness: pq - - coverage: - - postgres/client - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: interface - total: 9 - harness: postgres - - coverage: - - postgres/interface - - postgres/interface/page - - # ******************************************************************************************************************************** - name: protocol test: @@ -483,6 +464,7 @@ total: 6 coverage: + - config/common - config/config - config/parse - config/parse.auto: noCode @@ -594,6 +576,27 @@ - storage/write # ******************************************************************************************************************************** + - name: postgres + + test: + # ---------------------------------------------------------------------------------------------------------------------------- + - name: client + total: 1 + harness: pq + + coverage: + - postgres/client + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: interface + total: 9 + harness: postgres + + coverage: + - postgres/interface + - postgres/interface/page + + # ******************************************************************************************************************************** - name: build test: @@ -770,7 +773,7 @@ # ---------------------------------------------------------------------------------------------------------------------------- - name: help - total: 5 + total: 4 coverage: - command/help/help @@ -925,11 +928,11 @@ # ---------------------------------------------------------------------------------------------------------------------------- - name: archive-perl total: 1 + binReq: true - test: # ---------------------------------------------------------------------------------------------------------------------------- - name: type - total: 5 + total: 6 # ---------------------------------------------------------------------------------------------------------------------------- - name: storage diff -Nru pgbackrest-2.36/test/expect/mock-all-001.log pgbackrest-2.37/test/expect/mock-all-001.log --- pgbackrest-2.36/test/expect/mock-all-001.log 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/expect/mock-all-001.log 2022-01-03 13:43:55.000000000 +0000 @@ -1006,7 +1006,7 @@ P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2] P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2] P00 INFO: new backup label = [BACKUP-INCR-2] -P00 INFO: incr backup size = 192KB, file total = 22 +P00 INFO: incr backup size = 41B, file total = 22 P00 INFO: backup command end: completed successfully P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log --no-log-timestamp --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired @@ -1224,7 +1224,7 @@ P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2] P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2] P00 INFO: new backup label = [BACKUP-DIFF-1] -P00 INFO: diff backup size = 192KB, file total = 21 +P00 INFO: diff backup size = 32B, file total = 21 P00 INFO: backup command end: completed successfully P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log --no-log-timestamp --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired @@ -1800,7 +1800,7 @@ P00 DETAIL: reference pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt to [BACKUP-DIFF-1] P00 DETAIL: reference pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt to [BACKUP-INCR-3] P00 INFO: new backup label = [BACKUP-INCR-4] -P00 INFO: incr backup size = 176KB, file total = 22 +P00 INFO: incr backup size = 8B, file total = 22 P00 INFO: backup command end: completed successfully P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log --no-log-timestamp --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired @@ -2017,7 +2017,7 @@ P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2] P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2] P00 INFO: new backup label = [BACKUP-DIFF-2] -P00 INFO: diff backup size = 176KB, file total = 22 +P00 INFO: diff backup size = 46B, file total = 22 P00 INFO: backup command end: completed successfully P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log --no-log-timestamp --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired @@ -2234,7 +2234,7 @@ P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2] P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2] P00 INFO: new backup label = [BACKUP-DIFF-3] -P00 INFO: diff backup size = 176KB, file total = 20 +P00 INFO: diff backup size = 37B, file total = 20 P00 INFO: backup command end: completed successfully P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log --no-log-timestamp --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired diff -Nru pgbackrest-2.36/test/lib/pgBackRestTest/Common/CoverageTest.pm pgbackrest-2.37/test/lib/pgBackRestTest/Common/CoverageTest.pm --- pgbackrest-2.36/test/lib/pgBackRestTest/Common/CoverageTest.pm 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/lib/pgBackRestTest/Common/CoverageTest.pm 2022-01-03 13:43:55.000000000 +0000 @@ -39,7 +39,7 @@ my $bCoverageSummary = shift; my $strBranchFilter = - 'OBJECT_DEFINE_[A-Z0-9_]+\(|\s{4}[A-Z][A-Z0-9_]+\([^\?]*\)|\s{4}(ASSERT|assert|switch\s)\(|\{\+{0,1}' . + 'OBJECT_DEFINE_[A-Z0-9_]+\(|\s{4}[A-Z][A-Z0-9_]+\([^\?]*\)|\s{4}(ASSERT|CHECK|assert|switch\s)\(|\{\+{0,1}' . ($bCoverageSummary ? 'uncoverable_branch' : 'uncover(ed|able)_branch'); my $strLineFilter = '\{\+{0,1}' . ($bCoverageSummary ? 'uncoverable' : '(uncover(ed|able)' . ($bContainer ? '' : '|vm_covered') . ')') . '[^_]'; @@ -54,7 +54,7 @@ "#\n" . '# OBJECT_DEFINE_[A-Z0-9_]+\( - exclude object definitions' . "\n" . '# \s{4}[A-Z][A-Z0-9_]+\([^\?]*\) - exclude macros that do not take a conditional parameter and are not themselves a parameter' . "\n" . - '# ASSERT/(|assert\( - exclude asserts since it usually not possible to trigger both branches' . "\n" . + '# ASSERT/(|CHECK/(|assert\( - exclude asserts/checks since it usually not possible to trigger both branches' . "\n" . '# switch \( - lcov requires default: to show complete coverage but --Wswitch-enum enforces all enum values be present' . "\n" . "lcov_excl_br_line=${strBranchFilter}\n" . "\n" . diff -Nru pgbackrest-2.36/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm pgbackrest-2.37/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm --- pgbackrest-2.36/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm 2022-01-03 13:43:55.000000000 +0000 @@ -98,7 +98,7 @@ $strName, $strContainer, $$oParam{strImage}, $$oParam{strUser}, testRunGet()->vm(), ["${strProjectPath}:${strProjectPath}", "${strTestPath}:${strTestPath}", "${strBinPath}:${strBinPath}:ro"], undef, $oParam->{bTls} ? - 'server-start --log-level-console=debug --tls-server-ca-file=' . testRunGet()->basePath() . HOST_SERVER_CA . + 'server --log-level-console=debug --tls-server-ca-file=' . testRunGet()->basePath() . HOST_SERVER_CA . ' --tls-server-cert-file=' . testRunGet()->basePath() . HOST_SERVER_CERT . ' --tls-server-key-file=' . testRunGet()->basePath() . HOST_SERVER_KEY . ' --tls-server-auth=pgbackrest-client=* --tls-server-address=0.0.0.0' : undef, diff -Nru pgbackrest-2.36/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm pgbackrest-2.37/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm --- pgbackrest-2.36/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/lib/pgBackRestTest/Module/Performance/PerformanceArchivePerlTest.pm 2022-01-03 13:43:55.000000000 +0000 @@ -28,6 +28,7 @@ my $self = shift; $self->{strSpoolPath} = $self->testPath() . '/spool'; + $self->{strLogPath} = $self->testPath() . '/log'; } #################################################################################################################################### @@ -39,6 +40,7 @@ # Create spool path storageTest()->pathCreate($self->{strSpoolPath}, {bIgnoreExists => true, bCreateParent => true}); + storageTest()->pathCreate($self->{strLogPath}, {bIgnoreExists => true, bCreateParent => true}); } #################################################################################################################################### @@ -56,14 +58,20 @@ storageTest()->openWrite( 'spool/archive/' . $self->stanza() . '/out/000000010000000100000001.ok', {bPathCreate => true})); - my $iRunTotal = 1; + my $iRunTotal = 100; my $lTimeBegin = gettimeofday(); for (my $iIndex = 0; $iIndex < $iRunTotal; $iIndex++) { - executeTest( + my $iResult = system( $self->backrestExe() . ' --stanza=' . $self->stanza() . ' --archive-async --spool-path=' . $self->{strSpoolPath} . - ' --archive-timeout=1 archive-push /pg_xlog/000000010000000100000001'); + ' --log-level-file=detail --log-path=' . $self->{strLogPath} . ' --archive-timeout=1' . + ' --pg1-path=/not/used archive-push /pg_xlog/000000010000000100000001'); + + if ($iResult != 0) + { + confess "archive-push returned ${iResult}"; + } } &log(INFO, 'time per execution: ' . ((gettimeofday() - $lTimeBegin) / $iRunTotal)); diff -Nru pgbackrest-2.36/test/src/common/harnessLog.c pgbackrest-2.37/test/src/common/harnessLog.c --- pgbackrest-2.36/test/src/common/harnessLog.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/common/harnessLog.c 2022-01-03 13:43:55.000000000 +0000 @@ -371,7 +371,7 @@ int diff = (int)strSize(replace) - (int)strSize(match); // Make sure we won't overflow the buffer - CHECK((size_t)((int)strlen(harnessLogBuffer) + diff) < sizeof(harnessLogBuffer) - 1); + ASSERT((size_t)((int)strlen(harnessLogBuffer) + diff) < sizeof(harnessLogBuffer) - 1); // Move data from end of string enough to make room for the replacement and copy replacement memmove(end + diff, end, strlen(end) + 1); @@ -426,6 +426,9 @@ harnessLogLoad(logFile); hrnLogReplace(); + // Close expect log file + close(logFdFile); + if (strcmp(harnessLogBuffer, "") != 0) THROW_FMT(AssertError, "\n\nexpected log to be empty but actual log was:\n\n%s\n\n", harnessLogBuffer); diff -Nru pgbackrest-2.36/test/src/common/harnessPostgres/harnessVersion.intern.h pgbackrest-2.37/test/src/common/harnessPostgres/harnessVersion.intern.h --- pgbackrest-2.36/test/src/common/harnessPostgres/harnessVersion.intern.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/common/harnessPostgres/harnessVersion.intern.h 2022-01-03 13:43:55.000000000 +0000 @@ -45,6 +45,11 @@ .system_identifier = pgControl.systemId, \ .pg_control_version = PG_CONTROL_VERSION, \ .catalog_version_no = pgControl.catalogVersion, \ + .checkPoint = pgControl.checkpoint, \ + .checkPointCopy = \ + { \ + .ThisTimeLineID = pgControl.timeline, \ + }, \ .blcksz = pgControl.pageSize, \ .xlog_seg_size = pgControl.walSegmentSize, \ .data_checksum_version = pgControl.pageChecksum, \ @@ -65,6 +70,15 @@ .system_identifier = pgControl.systemId, \ .pg_control_version = PG_CONTROL_VERSION, \ .catalog_version_no = pgControl.catalogVersion, \ + .checkPoint = \ + { \ + .xlogid = (uint32_t)(pgControl.checkpoint >> 32), \ + .xrecoff = (uint32_t)(pgControl.checkpoint & 0xFFFFFFFF), \ + }, \ + .checkPointCopy = \ + { \ + .ThisTimeLineID = pgControl.timeline, \ + }, \ .blcksz = pgControl.pageSize, \ .xlog_seg_size = pgControl.walSegmentSize, \ }; \ diff -Nru pgbackrest-2.36/test/src/common/harnessPostgres.c pgbackrest-2.37/test/src/common/harnessPostgres.c --- pgbackrest-2.36/test/src/common/harnessPostgres.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/common/harnessPostgres.c 2022-01-03 13:43:55.000000000 +0000 @@ -230,11 +230,16 @@ FUNCTION_HARNESS_PARAM(PG_CONTROL, pgControl); FUNCTION_HARNESS_END(); + ASSERT(pgControl.version != 0); + // Set defaults if values are not passed pgControl.pageSize = pgControl.pageSize == 0 ? PG_PAGE_SIZE_DEFAULT : pgControl.pageSize; pgControl.walSegmentSize = pgControl.walSegmentSize == 0 ? PG_WAL_SEGMENT_SIZE_DEFAULT : pgControl.walSegmentSize; pgControl.catalogVersion = pgControl.catalogVersion == 0 ? hrnPgInterfaceVersion(pgControl.version)->catalogVersion() : pgControl.catalogVersion; + pgControl.systemId = pgControl.systemId < 100 ? hrnPgSystemId(pgControl.version) + pgControl.systemId : pgControl.systemId; + pgControl.checkpoint = pgControl.checkpoint == 0 ? 1 : pgControl.checkpoint; + pgControl.timeline = pgControl.timeline == 0 ? 1 : pgControl.timeline; // Create the buffer and clear it Buffer *result = bufNew(HRN_PG_CONTROL_SIZE); @@ -262,6 +267,10 @@ if (pgWal.size == 0) pgWal.size = PG_WAL_SEGMENT_SIZE_DEFAULT; + // Set default system id if not specified + if (pgWal.systemId < 100) + pgWal.systemId = hrnPgSystemId(pgWal.version) + pgWal.systemId; + // Generate WAL hrnPgInterfaceVersion(pgWal.version)->wal(pgWal, bufPtr(walBuffer)); diff -Nru pgbackrest-2.36/test/src/common/harnessPostgres.h pgbackrest-2.37/test/src/common/harnessPostgres.h --- pgbackrest-2.36/test/src/common/harnessPostgres.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/common/harnessPostgres.h 2022-01-03 13:43:55.000000000 +0000 @@ -7,12 +7,61 @@ #include "postgres/interface.h" #include "postgres/version.h" +#include "common/harnessStorage.h" + /*********************************************************************************************************************************** Control file size used to create pg_control ***********************************************************************************************************************************/ #define HRN_PG_CONTROL_SIZE 8192 /*********************************************************************************************************************************** +System id constants by version +***********************************************************************************************************************************/ +#define HRN_PG_SYSTEMID_83 (10000000000000000000ULL + (uint64_t)PG_VERSION_83) +#define HRN_PG_SYSTEMID_83_Z "10000000000000080300" +#define HRN_PG_SYSTEMID_84 (10000000000000000000ULL + (uint64_t)PG_VERSION_84) +#define HRN_PG_SYSTEMID_84_Z "10000000000000080400" +#define HRN_PG_SYSTEMID_90 (10000000000000000000ULL + (uint64_t)PG_VERSION_90) +#define HRN_PG_SYSTEMID_90_Z "10000000000000090000" +#define HRN_PG_SYSTEMID_91 (10000000000000000000ULL + (uint64_t)PG_VERSION_91) +#define HRN_PG_SYSTEMID_91_Z "10000000000000090100" +#define HRN_PG_SYSTEMID_92 (10000000000000000000ULL + (uint64_t)PG_VERSION_92) +#define HRN_PG_SYSTEMID_92_Z "10000000000000090200" +#define HRN_PG_SYSTEMID_93 (10000000000000000000ULL + (uint64_t)PG_VERSION_93) +#define HRN_PG_SYSTEMID_93_Z "10000000000000090300" +#define HRN_PG_SYSTEMID_94 (10000000000000000000ULL + (uint64_t)PG_VERSION_94) +#define HRN_PG_SYSTEMID_94_Z "10000000000000090400" +#define HRN_PG_SYSTEMID_95 (10000000000000000000ULL + (uint64_t)PG_VERSION_95) +#define HRN_PG_SYSTEMID_95_Z "10000000000000090500" +#define HRN_PG_SYSTEMID_96 (10000000000000000000ULL + (uint64_t)PG_VERSION_96) +#define HRN_PG_SYSTEMID_96_Z "10000000000000090600" +#define HRN_PG_SYSTEMID_10 (10000000000000000000ULL + (uint64_t)PG_VERSION_10) +#define HRN_PG_SYSTEMID_10_Z "10000000000000100000" +#define HRN_PG_SYSTEMID_10_1_Z "10000000000000100001" +#define HRN_PG_SYSTEMID_11 (10000000000000000000ULL + (uint64_t)PG_VERSION_11) +#define HRN_PG_SYSTEMID_11_Z "10000000000000110000" +#define HRN_PG_SYSTEMID_11_1_Z "10000000000000110001" +#define HRN_PG_SYSTEMID_12 (10000000000000000000ULL + (uint64_t)PG_VERSION_12) +#define HRN_PG_SYSTEMID_12_Z "10000000000000120000" +#define HRN_PG_SYSTEMID_13 (10000000000000000000ULL + (uint64_t)PG_VERSION_13) +#define HRN_PG_SYSTEMID_13_Z "10000000000000130000" +#define HRN_PG_SYSTEMID_14 (10000000000000000000ULL + (uint64_t)PG_VERSION_14) +#define HRN_PG_SYSTEMID_14_Z "10000000000000140000" + +/*********************************************************************************************************************************** +Put a control file to storage +***********************************************************************************************************************************/ +#define HRN_PG_CONTROL_PUT(storageParam, versionParam, ...) \ + HRN_STORAGE_PUT( \ + storageParam, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer((PgControl){.version = versionParam, __VA_ARGS__})) + +/*********************************************************************************************************************************** +Update control file time +***********************************************************************************************************************************/ +#define HRN_PG_CONTROL_TIME(storageParam, timeParam, ...) \ + HRN_STORAGE_TIME(storageParam, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, timeParam) + +/*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ // Get the catalog version for a PostgreSQL version @@ -21,6 +70,13 @@ // Create pg_control Buffer *hrnPgControlToBuffer(PgControl pgControl); +// Get system id by version +__attribute__((always_inline)) static inline uint64_t +hrnPgSystemId(const unsigned int pgVersion) +{ + return 10000000000000000000ULL + (uint64_t)pgVersion; +} + // Create WAL for testing void hrnPgWalToBuffer(PgWal pgWal, Buffer *walBuffer); diff -Nru pgbackrest-2.36/test/src/common/harnessPq.h pgbackrest-2.37/test/src/common/harnessPq.h --- pgbackrest-2.36/test/src/common/harnessPq.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/common/harnessPq.h 2022-01-03 13:43:55.000000000 +0000 @@ -68,24 +68,27 @@ "[\"select (select setting from pg_catalog.pg_settings where name = 'server_version_num')::int4," \ " (select setting from pg_catalog.pg_settings where name = 'data_directory')::text," \ " (select setting from pg_catalog.pg_settings where name = 'archive_mode')::text," \ - " (select setting from pg_catalog.pg_settings where name = 'archive_command')::text\"]", \ + " (select setting from pg_catalog.pg_settings where name = 'archive_command')::text," \ + " (select setting from pg_catalog.pg_settings where name = 'checkpoint_timeout')::int4\"]", \ .resultInt = 1}, \ {.session = sessionParam, .function = HRNPQ_CONSUMEINPUT}, \ {.session = sessionParam, .function = HRNPQ_ISBUSY}, \ {.session = sessionParam, .function = HRNPQ_GETRESULT}, \ {.session = sessionParam, .function = HRNPQ_RESULTSTATUS, .resultInt = PGRES_TUPLES_OK}, \ {.session = sessionParam, .function = HRNPQ_NTUPLES, .resultInt = 1}, \ - {.session = sessionParam, .function = HRNPQ_NFIELDS, .resultInt = 4}, \ + {.session = sessionParam, .function = HRNPQ_NFIELDS, .resultInt = 5}, \ {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[0]", .resultInt = HRNPQ_TYPE_INT}, \ {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[1]", .resultInt = HRNPQ_TYPE_TEXT}, \ {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[2]", .resultInt = HRNPQ_TYPE_TEXT}, \ {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[3]", .resultInt = HRNPQ_TYPE_TEXT}, \ + {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[4]", .resultInt = HRNPQ_TYPE_INT}, \ {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,0]", .resultZ = STRINGIFY(versionParam)}, \ {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,1]", .resultZ = pgPathParam}, \ {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,2]", .resultZ = archiveMode == NULL ? "on" \ : archiveMode}, \ {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,3]", .resultZ = archiveCommand == NULL ? PROJECT_BIN \ : archiveCommand}, \ + {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,4]", .resultZ = STRINGIFY(300)}, \ {.session = sessionParam, .function = HRNPQ_CLEAR}, \ {.session = sessionParam, .function = HRNPQ_GETRESULT, .resultNull = true} @@ -138,6 +141,36 @@ {.session = sessionParam, .function = HRNPQ_CLEAR}, \ {.session = sessionParam, .function = HRNPQ_GETRESULT, .resultNull = true} +#define HRNPQ_MACRO_CURRENT_WAL_LE_96(sessionParam, walSegmentParam) \ + {.session = sessionParam, \ + .function = HRNPQ_SENDQUERY, \ + .param = "[\"select pg_catalog.pg_xlogfile_name(pg_catalog.pg_current_xlog_insert_location())::text\"]", .resultInt = 1}, \ + {.session = sessionParam, .function = HRNPQ_CONSUMEINPUT}, \ + {.session = sessionParam, .function = HRNPQ_ISBUSY}, \ + {.session = sessionParam, .function = HRNPQ_GETRESULT}, \ + {.session = sessionParam, .function = HRNPQ_RESULTSTATUS, .resultInt = PGRES_TUPLES_OK}, \ + {.session = sessionParam, .function = HRNPQ_NTUPLES, .resultInt = 1}, \ + {.session = sessionParam, .function = HRNPQ_NFIELDS, .resultInt = 1}, \ + {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[0]", .resultInt = HRNPQ_TYPE_TEXT}, \ + {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,0]", .resultZ = walSegmentParam}, \ + {.session = sessionParam, .function = HRNPQ_CLEAR}, \ + {.session = sessionParam, .function = HRNPQ_GETRESULT, .resultNull = true} + +#define HRNPQ_MACRO_CURRENT_WAL_GE_10(sessionParam, walSegmentParam) \ + {.session = sessionParam, \ + .function = HRNPQ_SENDQUERY, \ + .param = "[\"select pg_catalog.pg_walfile_name(pg_catalog.pg_current_wal_insert_lsn())::text\"]", .resultInt = 1}, \ + {.session = sessionParam, .function = HRNPQ_CONSUMEINPUT}, \ + {.session = sessionParam, .function = HRNPQ_ISBUSY}, \ + {.session = sessionParam, .function = HRNPQ_GETRESULT}, \ + {.session = sessionParam, .function = HRNPQ_RESULTSTATUS, .resultInt = PGRES_TUPLES_OK}, \ + {.session = sessionParam, .function = HRNPQ_NTUPLES, .resultInt = 1}, \ + {.session = sessionParam, .function = HRNPQ_NFIELDS, .resultInt = 1}, \ + {.session = sessionParam, .function = HRNPQ_FTYPE, .param = "[0]", .resultInt = HRNPQ_TYPE_TEXT}, \ + {.session = sessionParam, .function = HRNPQ_GETVALUE, .param = "[0,0]", .resultZ = walSegmentParam}, \ + {.session = sessionParam, .function = HRNPQ_CLEAR}, \ + {.session = sessionParam, .function = HRNPQ_GETRESULT, .resultNull = true} + #define HRNPQ_MACRO_WAL_SWITCH(sessionParam, walNameParam, walFileNameParam) \ {.session = sessionParam, .function = HRNPQ_SENDQUERY, \ .param = "[\"select pg_catalog.pg_" walNameParam "file_name(pg_catalog.pg_switch_" walNameParam "())::text\"]", \ diff -Nru pgbackrest-2.36/test/src/common/harnessTest.h pgbackrest-2.37/test/src/common/harnessTest.h --- pgbackrest-2.36/test/src/common/harnessTest.h 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/common/harnessTest.h 2022-01-03 13:43:55.000000000 +0000 @@ -6,6 +6,7 @@ #include +#include "common/assert.h" #include "common/debug.h" #include "common/error.h" @@ -17,6 +18,21 @@ #define BOGUS_STR "BOGUS" /*********************************************************************************************************************************** +Make sure ASSERT() always exists for tests to use, even when DEBUG is disabled for performance +***********************************************************************************************************************************/ +#ifdef HRN_FEATURE_ASSERT + #undef ASSERT + + #define ASSERT(condition) \ + do \ + { \ + if (!(condition)) \ + THROW_FMT(AssertError, "assertion '%s' failed", #condition); \ + } \ + while (0) +#endif + +/*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ // Begin a test if this function returns true, otherwise the user has skipped it diff -Nru pgbackrest-2.36/test/src/module/build/configTest.c pgbackrest-2.37/test/src/module/build/configTest.c --- pgbackrest-2.36/test/src/module/build/configTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/build/configTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -250,6 +250,16 @@ " command-role:\n" " main: {}\n" "\n" + " compress-network:\n" + " section: global\n" + " type: string-id\n" + " command:\n" + " backup:\n" + " allow-list:\n" + " - gz\n" + " command-role:\n" + " main: {}\n" + "\n" " compress-type:\n" " section: global\n" " type: string\n" @@ -301,7 +311,7 @@ "\n" " log-level-console:\n" " section: global\n" - " type: string\n" + " type: string-id\n" " default: warn\n" " allow-list:\n" " - off\n" @@ -311,7 +321,7 @@ "\n" " log-level-file:\n" " section: global\n" - " type: string\n" + " type: string-id\n" " default: info\n" " allow-list: log-level-console\n" " command:\n" @@ -429,6 +439,7 @@ "#define CFGOPT_BUFFER_SIZE \"buffer-size\"\n" "#define CFGOPT_COMPRESS_LEVEL \"compress-level\"\n" "#define CFGOPT_COMPRESS_LEVEL_NETWORK \"compress-level-network\"\n" + "#define CFGOPT_COMPRESS_NETWORK \"compress-network\"\n" "#define CFGOPT_COMPRESS_TYPE \"compress-type\"\n" "#define CFGOPT_CONFIG \"config\"\n" "#define CFGOPT_CONFIG_INCLUDE \"config-include\"\n" @@ -438,11 +449,14 @@ "#define CFGOPT_STANZA \"stanza\"\n" "#define CFGOPT_TIMEOUT \"timeout\"\n" "\n" - "#define CFG_OPTION_TOTAL 14\n" + "#define CFG_OPTION_TOTAL 15\n" "\n" COMMENT_BLOCK_BEGIN "\n" "Option value constants\n" COMMENT_BLOCK_END "\n" + "#define CFGOPTVAL_COMPRESS_NETWORK_GZ STRID5(\"gz\", 0x3470)\n" + "#define CFGOPTVAL_COMPRESS_NETWORK_GZ_Z \"gz\"\n" + "\n" "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_DEBUG1 STRID6(\"debug1\", 0x7475421441)\n" "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_DEBUG1_Z \"debug1\"\n" "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_ERROR STRID5(\"error\", 0x127ca450)\n" @@ -491,6 +505,7 @@ " cfgOptBufferSize,\n" " cfgOptCompressLevel,\n" " cfgOptCompressLevelNetwork,\n" + " cfgOptCompressNetwork,\n" " cfgOptCompressType,\n" " cfgOptConfig,\n" " cfgOptConfigInclude,\n" @@ -561,6 +576,7 @@ "{\n" " STRID6(\"debug1\", 0x7475421441),\n" " STRID5(\"error\", 0x127ca450),\n" + " STRID5(\"gz\", 0x3470),\n" " STRID5(\"info\", 0x799c90),\n" " STRID5(\"off\", 0x18cf0),\n" " STRID5(\"warn\", 0x748370),\n" @@ -570,6 +586,7 @@ "{\n" " parseRuleValStrIdDebug1,\n" " parseRuleValStrIdError,\n" + " parseRuleValStrIdGz,\n" " parseRuleValStrIdInfo,\n" " parseRuleValStrIdOff,\n" " parseRuleValStrIdWarn,\n" @@ -923,6 +940,37 @@ COMMENT_SEPARATOR "\n" " PARSE_RULE_OPTION\n" " (\n" + " PARSE_RULE_OPTION_NAME(\"compress-network\"),\n" + " PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId),\n" + " PARSE_RULE_OPTION_RESET(true),\n" + " PARSE_RULE_OPTION_REQUIRED(true),\n" + " PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),\n" + "\n" + " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" + " (\n" + " PARSE_RULE_OPTION_COMMAND(cfgCmdBackup)\n" + " ),\n" + "\n" + " PARSE_RULE_OPTIONAL\n" + " (\n" + " PARSE_RULE_OPTIONAL_GROUP\n" + " (\n" + " PARSE_RULE_FILTER_CMD\n" + " (\n" + " PARSE_RULE_VAL_CMD(cfgCmdBackup),\n" + " ),\n" + "\n" + " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" + " (\n" + " PARSE_RULE_VAL_STRID(parseRuleValStrIdGz),\n" + " ),\n" + " ),\n" + " ),\n" + " ),\n" + "\n" + COMMENT_SEPARATOR "\n" + " PARSE_RULE_OPTION\n" + " (\n" " PARSE_RULE_OPTION_NAME(\"compress-type\"),\n" " PARSE_RULE_OPTION_TYPE(cfgOptTypeString),\n" " PARSE_RULE_OPTION_RESET(true),\n" @@ -1047,7 +1095,7 @@ " PARSE_RULE_OPTION\n" " (\n" " PARSE_RULE_OPTION_NAME(\"log-level-console\"),\n" - " PARSE_RULE_OPTION_TYPE(cfgOptTypeString),\n" + " PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId),\n" " PARSE_RULE_OPTION_RESET(true),\n" " PARSE_RULE_OPTION_REQUIRED(true),\n" " PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),\n" @@ -1089,6 +1137,7 @@ "\n" " PARSE_RULE_OPTIONAL_DEFAULT\n" " (\n" + " PARSE_RULE_VAL_STRID(parseRuleValStrIdWarn),\n" " PARSE_RULE_VAL_STR(parseRuleValStrQT_warn_QT),\n" " ),\n" " ),\n" @@ -1099,7 +1148,7 @@ " PARSE_RULE_OPTION\n" " (\n" " PARSE_RULE_OPTION_NAME(\"log-level-file\"),\n" - " PARSE_RULE_OPTION_TYPE(cfgOptTypeString),\n" + " PARSE_RULE_OPTION_TYPE(cfgOptTypeStringId),\n" " PARSE_RULE_OPTION_RESET(true),\n" " PARSE_RULE_OPTION_REQUIRED(true),\n" " PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),\n" @@ -1149,6 +1198,7 @@ "\n" " PARSE_RULE_OPTIONAL_DEFAULT\n" " (\n" + " PARSE_RULE_VAL_STRID(parseRuleValStrIdWarn),\n" " PARSE_RULE_VAL_STR(parseRuleValStrQT_warn_QT),\n" " ),\n" "\n" @@ -1167,6 +1217,7 @@ "\n" " PARSE_RULE_OPTIONAL_DEFAULT\n" " (\n" + " PARSE_RULE_VAL_STRID(parseRuleValStrIdInfo),\n" " PARSE_RULE_VAL_STR(parseRuleValStrQT_info_QT),\n" " ),\n" " ),\n" @@ -1488,6 +1539,7 @@ "{\n" " cfgOptStanza,\n" " cfgOptBufferSize,\n" + " cfgOptCompressNetwork,\n" " cfgOptConfig,\n" " cfgOptConfigInclude,\n" " cfgOptLogLevelConsole,\n" diff -Nru pgbackrest-2.36/test/src/module/command/archiveGetTest.c pgbackrest-2.37/test/src/module/command/archiveGetTest.c --- pgbackrest-2.36/test/src/module/command/archiveGetTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/archiveGetTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -150,9 +150,7 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("no segments to find"); - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_10); HRN_INFO_PUT( storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, @@ -160,7 +158,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n"); strLstAddZ(argList, "000000010000000100000001"); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .role = cfgCmdRoleAsync); @@ -252,8 +250,8 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n" - "2={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n" + "2={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n"); HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"); @@ -290,8 +288,8 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n" - "2={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n" + "2={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n"); HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"); @@ -336,7 +334,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}\n"); HRN_STORAGE_PUT_EMPTY( storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/0000000100000001000000FE-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"); @@ -352,7 +350,7 @@ TEST_RESULT_LOG( "P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000\n" "P00 WARN: repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'\n" + " '" HRN_PG_SYSTEMID_10_Z "'\n" "P01 DETAIL: found 0000000100000001000000FE in the repo1: 10-1 archive\n" "P00 DETAIL: unable to find 0000000100000001000000FF in the archive"); @@ -360,13 +358,13 @@ storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE.ok", "0\n" "repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'", + " '" HRN_PG_SYSTEMID_10_Z "'", .remove = true); TEST_STORAGE_GET( storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF.ok", "0\n" "repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'", + " '" HRN_PG_SYSTEMID_10_Z "'", .remove = true); TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", .remove = true); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN); @@ -381,7 +379,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n"); HRN_STORAGE_PATH_CREATE(storageRepoIdxWrite(1), STORAGE_REPO_ARCHIVE "/10-1", .mode = 0400); @@ -479,7 +477,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"11\"}\n"); HRN_STORAGE_PATH_CREATE(storageRepoIdxWrite(2), "10-1", .mode = 0400); HRN_STORAGE_PUT_EMPTY( @@ -494,7 +492,7 @@ TEST_RESULT_LOG( "P00 INFO: get 1 WAL file(s) from archive: 000000010000000200000000\n" "P00 WARN: repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'\n" + " '" HRN_PG_SYSTEMID_10_Z "'\n" "P01 WARN: repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz" " [FormatError] unexpected eof in compressed data\n" "P01 DETAIL: found 000000010000000200000000 in the repo2: 10-1 archive"); @@ -503,7 +501,7 @@ storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.ok", "0\n" "repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'\n" + " '" HRN_PG_SYSTEMID_10_Z "'\n" "repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz" " [FormatError] unexpected eof in compressed data", .remove = true); @@ -528,7 +526,7 @@ TEST_RESULT_LOG( "P00 INFO: get 1 WAL file(s) from archive: 000000010000000200000000\n" "P00 WARN: repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'\n" + " '" HRN_PG_SYSTEMID_10_Z "'\n" "P01 WARN: [FileReadError] raised from local-1 shim protocol: unable to get 000000010000000200000000:\n" " repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz" " [FormatError] unexpected eof in compressed data\n" @@ -546,7 +544,7 @@ " [FormatError] unexpected eof in compressed data\n" "[FileReadError] on retry after 0ms\n" "repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '18072658121562454734'", + " '" HRN_PG_SYSTEMID_10_Z "'", .remove = true); TEST_STORAGE_LIST( storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000200000000.pgbackrest.tmp\n", .remove = true); @@ -624,9 +622,7 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("no valid repo"); - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_10); strLstAddZ(argList, TEST_PATH "/pg/pg_wal/RECOVERYXLOG"); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true); @@ -782,9 +778,7 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("pg version does not match archive.info"); - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_11); HRN_INFO_PUT( storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, @@ -792,7 +786,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}"); argBaseList = strLstNew(); hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH "/pg"); @@ -808,27 +802,23 @@ TEST_RESULT_LOG( "P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '11' and system-id" - " '18072658121562454734'"); + " '" HRN_PG_SYSTEMID_11_Z "'"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("pg system id does not match archive.info"); - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0x8888888888888888})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_10, .systemId = 1); TEST_ERROR(cmdArchiveGet(), RepoInvalidError, "unable to find a valid repository"); TEST_RESULT_LOG( "P00 WARN: repo1: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" - " '9838263505978427528'"); + " '" HRN_PG_SYSTEMID_10_1_Z "'"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("file is missing"); - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_10); TEST_RESULT_INT(cmdArchiveGet(), 1, "get"); @@ -881,10 +871,10 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n" - "2={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n" - "3={\"db-id\":10000000000000000000,\"db-version\":\"11\"}\n" - "4={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n" + "2={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}\n" + "3={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}\n" + "4={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}"); TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); @@ -974,7 +964,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}", + "1={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}", .cipherType = cipherTypeAes256Cbc); HRN_STORAGE_PUT( @@ -1022,7 +1012,7 @@ "db-id=2\n" "\n" "[db:history]\n" - "2={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); + "2={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}"); HRN_STORAGE_PATH_CREATE(storageRepoIdxWrite(0), STORAGE_REPO_ARCHIVE "/10-2", .mode = 0400); @@ -1117,7 +1107,7 @@ "db-id=2\n" "\n" "[db:history]\n" - "2={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); + "2={\"db-id\":" HRN_PG_SYSTEMID_10_Z ",\"db-version\":\"10\"}"); // Put a warning in the file to show that it was read and later overwritten HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok", "0\nshould not be output"); diff -Nru pgbackrest-2.36/test/src/module/command/archivePushTest.c pgbackrest-2.37/test/src/module/command/archivePushTest.c --- pgbackrest-2.36/test/src/module/command/archivePushTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/archivePushTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -72,7 +72,7 @@ Buffer *walBuffer = bufNew((size_t)16 * 1024 * 1024); bufUsedSet(walBuffer, bufSize(walBuffer)); memset(bufPtr(walBuffer), 0, bufSize(walBuffer)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10}, walBuffer); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000002", walBuffer); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000003", walBuffer); @@ -104,9 +104,7 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("mismatched pg_control and archive.info - pg version"); - HRN_STORAGE_PUT( - storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 0xFACEFACEFACEFACE})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_96); // Create incorrect archive info HRN_INFO_PUT( @@ -120,8 +118,8 @@ TEST_ERROR( archivePushCheck(true), RepoInvalidError, "unable to find a valid repository:\n" - "repo1: [ArchiveMismatchError] PostgreSQL version 9.6, system-id 18072658121562454734 do not match repo1 stanza version" - " 9.4, system-id 5555555555555555555" + "repo1: [ArchiveMismatchError] PostgreSQL version 9.6, system-id " HRN_PG_SYSTEMID_96_Z " do not match repo1 stanza" + " version 9.4, system-id 5555555555555555555" "\nHINT: are you archiving to the correct stanza?"); // ------------------------------------------------------------------------------------------------------------------------- @@ -139,8 +137,8 @@ TEST_ERROR( archivePushCheck(true), RepoInvalidError, "unable to find a valid repository:\n" - "repo1: [ArchiveMismatchError] PostgreSQL version 9.6, system-id 18072658121562454734 do not match repo1 stanza version" - " 9.6, system-id 5555555555555555555" + "repo1: [ArchiveMismatchError] PostgreSQL version 9.6, system-id " HRN_PG_SYSTEMID_96_Z " do not match repo1 stanza" + " version 9.6, system-id 5555555555555555555" "\nHINT: are you archiving to the correct stanza?"); // ------------------------------------------------------------------------------------------------------------------------- @@ -153,13 +151,13 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n"); ArchivePushCheckResult result = {0}; TEST_ASSIGN(result, archivePushCheck(true), "get archive check result"); TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version"); - TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id"); + TEST_RESULT_UINT(result.pgSystemId, HRN_PG_SYSTEMID_96, "check pg system id"); ArchivePushFileRepoData *repoData = lstGet(result.repoList, 0); TEST_RESULT_UINT(repoData->repoIdx, 0, "check repo idx"); @@ -183,7 +181,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n"); // repo4 has incorrect info HRN_INFO_PUT( @@ -197,10 +195,10 @@ TEST_ASSIGN(result, archivePushCheck(false), "get archive check result"); TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version"); - TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id"); + TEST_RESULT_UINT(result.pgSystemId, HRN_PG_SYSTEMID_96, "check pg system id"); TEST_RESULT_STRLST_Z( result.errorList, - "repo4: [ArchiveMismatchError] repo2 stanza version 9.6, system-id 18072658121562454734 do not match repo4 stanza" + "repo4: [ArchiveMismatchError] repo2 stanza version 9.6, system-id " HRN_PG_SYSTEMID_96_Z " do not match repo4 stanza" " version 9.4, system-id 5555555555555555555\n" "HINT: are you archiving to the correct stanza?\n", "check error list"); @@ -222,12 +220,12 @@ "\n" "[db:history]\n" "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n" - "2={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"); + "2={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n"); TEST_ASSIGN(result, archivePushCheck(false), "get archive check result"); TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version"); - TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id"); + TEST_RESULT_UINT(result.pgSystemId, HRN_PG_SYSTEMID_96, "check pg system id"); repoData = lstGet(result.repoList, 0); TEST_RESULT_UINT(repoData->repoIdx, 0, "check repo idx"); @@ -289,9 +287,7 @@ strLstAddZ(argListTemp, "pg_wal/000000010000000100000001"); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); - HRN_STORAGE_PUT( - storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_11); HRN_INFO_PUT( storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, @@ -299,13 +295,13 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}\n"); // Generate WAL with incorrect headers and try to push them Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024); bufUsedSet(walBuffer1, bufSize(walBuffer1)); memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer1); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10}, walBuffer1); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1); @@ -313,19 +309,19 @@ TEST_ERROR( cmdArchivePush(), ArchiveMismatchError, - "WAL file '" TEST_PATH "/pg/pg_wal/000000010000000100000001' version 10, system-id 18072658121562454734 do not match" - " stanza version 11, system-id 18072658121562454734"); + "WAL file '" TEST_PATH "/pg/pg_wal/000000010000000100000001' version 10, system-id " HRN_PG_SYSTEMID_10_Z " do not" + " match stanza version 11, system-id " HRN_PG_SYSTEMID_11_Z ""); memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 1}, walBuffer1); const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1))); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1); TEST_ERROR( cmdArchivePush(), ArchiveMismatchError, - "WAL file '" TEST_PATH "/pg/pg_wal/000000010000000100000001' version 11, system-id 17055110554209741999 do not match" - " stanza version 11, system-id 18072658121562454734"); + "WAL file '" TEST_PATH "/pg/pg_wal/000000010000000100000001' version 11, system-id " HRN_PG_SYSTEMID_11_1_Z " do not" + " match stanza version 11, system-id " HRN_PG_SYSTEMID_11_Z); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("push by ignoring the invalid header"); @@ -350,13 +346,13 @@ HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer1); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11}, walBuffer1); // Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them // directly from the buffers to reduce the cost of maintaining checksums. walBuffer1Sha1 = TEST_64BIT() ? - (TEST_BIG_ENDIAN() ? "1c5f963d720bb199d7935dbd315447ea2ec3feb2" : "aae7591a1dbc58f21d0d004886075094f622e6dd") : - "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a"; + (TEST_BIG_ENDIAN() ? "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" : "858a9ef24b79468eb2a61543b58140addfede0fc") : + "044ec0576dc4e59d460aa3a8ac796ba4874ddff3"; HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1); @@ -377,7 +373,7 @@ Buffer *walBuffer2 = bufNew((size_t)16 * 1024 * 1024); bufUsedSet(walBuffer2, bufSize(walBuffer2)); memset(bufPtr(walBuffer2), 0xFF, bufSize(walBuffer2)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer2); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11}, walBuffer2); const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2))); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer2); @@ -487,7 +483,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}", + "1={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}", .cipherType = cipherTypeAes256Cbc, .cipherPass = "badpassphrase"); // repo3 is not encrypted @@ -497,7 +493,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}"); + "1={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}"); // Push encrypted WAL segment TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); @@ -671,8 +667,7 @@ hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); hrnCfgArgRawBool(argList, cfgOptLogSubprocess, true); - HRN_STORAGE_PUT(storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_94); HRN_INFO_PUT( storageTest, "repo/archive/test/archive.info", @@ -680,7 +675,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_94_Z ",\"db-version\":\"9.4\"}\n"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("async, ignore error file on first pass"); @@ -755,7 +750,7 @@ Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024); bufUsedSet(walBuffer1, bufSize(walBuffer1)); memset(bufPtr(walBuffer1), 0xFF, bufSize(walBuffer1)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer1); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94}, walBuffer1); const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1))); HRN_STORAGE_PUT(storagePgWrite(),"pg_xlog/000000010000000100000001", walBuffer1); @@ -816,7 +811,7 @@ "db-id=1\n" "\n" "[db:history]\n" - "1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_94_Z ",\"db-version\":\"9.4\"}\n"); // Recreate ready file for WAL 1 HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000001.ready"); @@ -860,7 +855,7 @@ Buffer *walBuffer2 = bufNew((size_t)16 * 1024 * 1024); bufUsedSet(walBuffer2, bufSize(walBuffer2)); memset(bufPtr(walBuffer2), 0x0C, bufSize(walBuffer2)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer2); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94}, walBuffer2); const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2))); HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000002", walBuffer2); @@ -909,7 +904,7 @@ Buffer *walBuffer3 = bufNew((size_t)16 * 1024 * 1024); bufUsedSet(walBuffer3, bufSize(walBuffer3)); memset(bufPtr(walBuffer3), 0x44, bufSize(walBuffer3)); - hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer3); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94}, walBuffer3); const char *walBuffer3Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer3))); HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000003", walBuffer3); diff -Nru pgbackrest-2.36/test/src/module/command/backupTest.c pgbackrest-2.37/test/src/module/command/backupTest.c --- pgbackrest-2.36/test/src/module/command/backupTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/backupTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -238,6 +238,8 @@ bool backupStandby; bool errorAfterStart; bool noWal; // Don't write test WAL segments + bool noPriorWal; // Don't write prior test WAL segments + bool noArchiveCheck; // Do not check archive CompressType walCompressType; // Compress type for the archive files unsigned int walTotal; // Total WAL to write unsigned int timeline; // Timeline to use for WAL files @@ -261,23 +263,39 @@ // Set archive timeout really small to save time on errors cfgOptionSet(cfgOptArchiveTimeout, cfgSourceParam, varNewInt64(100)); + // Set LSN and WAL start/stop uint64_t lsnStart = ((uint64_t)backupTimeStart & 0xFFFFFF00) << 28; uint64_t lsnStop = lsnStart + ((param.walTotal == 0 ? 0 : param.walTotal - 1) * pgControl.walSegmentSize) + (pgControl.walSegmentSize / 2); + const char *walSegmentPrior = strZ( + pgLsnToWalSegment(param.timeline, lsnStart - pgControl.walSegmentSize, pgControl.walSegmentSize)); const char *lsnStartStr = strZ(pgLsnToStr(lsnStart)); const char *walSegmentStart = strZ(pgLsnToWalSegment(param.timeline, lsnStart, pgControl.walSegmentSize)); const char *lsnStopStr = strZ(pgLsnToStr(lsnStop)); const char *walSegmentStop = strZ(pgLsnToWalSegment(param.timeline, lsnStop, pgControl.walSegmentSize)); + // Save pg_control with updated info + pgControl.checkpoint = lsnStart; + pgControl.timeline = param.timeline; + + HRN_STORAGE_PUT( + storagePgIdxWrite(0), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer(pgControl), + .timeModified = backupTimeStart); + + // Update pg_control on primary with the backup time + HRN_PG_CONTROL_TIME(storagePgIdxWrite(0), backupTimeStart); + // Write WAL segments to the archive // ----------------------------------------------------------------------------------------------------------------------------- - if (!param.noWal) + if (!param.noPriorWal) { InfoArchive *infoArchive = infoArchiveLoadFile(storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cipherTypeNone, NULL); const String *archiveId = infoArchiveId(infoArchive); StringList *walSegmentList = pgLsnRangeToWalSegmentList( - pgControl.version, param.timeline, lsnStart, lsnStop, pgControl.walSegmentSize); + pgControl.version, param.timeline, lsnStart - pgControl.walSegmentSize, + param.noWal ? lsnStart - pgControl.walSegmentSize : lsnStop, + pgControl.walSegmentSize); Buffer *walBuffer = bufNew((size_t)pgControl.walSegmentSize); bufUsedSet(walBuffer, bufSize(walBuffer)); @@ -306,77 +324,160 @@ ASSERT(!param.backupStandby); ASSERT(!param.errorAfterStart); - harnessPqScriptSet((HarnessPq []) + if (param.noArchiveCheck) { - // Connect to primary - HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_95, pg1Path, false, NULL, NULL), + harnessPqScriptSet((HarnessPq []) + { + // Connect to primary + HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_95, pg1Path, false, NULL, NULL), - // Get start time - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000), + // Get start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000), - // Start backup - HRNPQ_MACRO_ADVISORY_LOCK(1, true), - HRNPQ_MACRO_IS_IN_BACKUP(1, false), - HRNPQ_MACRO_START_BACKUP_84_95(1, param.startFast, lsnStartStr, walSegmentStart), - HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), - HRNPQ_MACRO_TABLESPACE_LIST_0(1), - - // Get copy start time - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999), - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000), + // Start backup + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_IS_IN_BACKUP(1, false), + HRNPQ_MACRO_START_BACKUP_84_95(1, param.startFast, lsnStartStr, walSegmentStart), + HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), + HRNPQ_MACRO_TABLESPACE_LIST_0(1), - // Stop backup - HRNPQ_MACRO_STOP_BACKUP_LE_95(1, lsnStopStr, walSegmentStop), + // Get copy start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999), + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000), - // Get stop time - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000), + // Ping + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), - HRNPQ_MACRO_DONE() - }); + // Stop backup + HRNPQ_MACRO_STOP_BACKUP_LE_95(1, lsnStopStr, walSegmentStop), + + // Get stop time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000), + + HRNPQ_MACRO_DONE() + }); + } + else + { + harnessPqScriptSet((HarnessPq []) + { + // Connect to primary + HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_95, pg1Path, false, NULL, NULL), + + // Get start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000), + + // Start backup + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_IS_IN_BACKUP(1, false), + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, walSegmentPrior), + HRNPQ_MACRO_START_BACKUP_84_95(1, param.startFast, lsnStartStr, walSegmentStart), + HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), + HRNPQ_MACRO_TABLESPACE_LIST_0(1), + + // Get copy start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999), + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000), + + // Ping + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + + // Stop backup + HRNPQ_MACRO_STOP_BACKUP_LE_95(1, lsnStopStr, walSegmentStop), + + // Get stop time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000), + + HRNPQ_MACRO_DONE() + }); + } } // ----------------------------------------------------------------------------------------------------------------------------- else if (pgVersion == PG_VERSION_96) { ASSERT(param.backupStandby); ASSERT(!param.errorAfterStart); + ASSERT(!param.noArchiveCheck); - harnessPqScriptSet((HarnessPq []) + // Save pg_control with updated info + HRN_STORAGE_PUT(storagePgIdxWrite(1), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer(pgControl)); + + if (param.noPriorWal) { - // Connect to primary - HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_96, pg1Path, false, NULL, NULL), + harnessPqScriptSet((HarnessPq []) + { + // Connect to primary + HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_96, pg1Path, false, NULL, NULL), + + // Connect to standby + HRNPQ_MACRO_OPEN_GE_96(2, "dbname='postgres' port=5433", PG_VERSION_96, pg2Path, true, NULL, NULL), + + // Get start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000), + + // Start backup + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, walSegmentPrior), + HRNPQ_MACRO_START_BACKUP_96(1, true, lsnStartStr, walSegmentStart), + HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), + HRNPQ_MACRO_TABLESPACE_LIST_0(1), + + // Wait for standby to sync + HRNPQ_MACRO_REPLAY_WAIT_96(2, lsnStartStr), + + HRNPQ_MACRO_DONE() + }); + } + else + { + harnessPqScriptSet((HarnessPq []) + { + // Connect to primary + HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_96, pg1Path, false, NULL, NULL), - // Connect to standby - HRNPQ_MACRO_OPEN_GE_96(2, "dbname='postgres' port=5433", PG_VERSION_96, pg2Path, true, NULL, NULL), + // Connect to standby + HRNPQ_MACRO_OPEN_GE_96(2, "dbname='postgres' port=5433", PG_VERSION_96, pg2Path, true, NULL, NULL), - // Get start time - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000), + // Get start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000), + + // Start backup + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, walSegmentPrior), + HRNPQ_MACRO_START_BACKUP_96(1, true, lsnStartStr, walSegmentStart), + HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), + HRNPQ_MACRO_TABLESPACE_LIST_0(1), - // Start backup - HRNPQ_MACRO_ADVISORY_LOCK(1, true), - HRNPQ_MACRO_START_BACKUP_96(1, true, lsnStartStr, walSegmentStart), - HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), - HRNPQ_MACRO_TABLESPACE_LIST_0(1), + // Wait for standby to sync + HRNPQ_MACRO_REPLAY_WAIT_96(2, lsnStartStr), - // Wait for standby to sync - HRNPQ_MACRO_REPLAY_WAIT_96(2, lsnStartStr), + // Get copy start time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999), + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000), - // Get copy start time - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999), - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000), + // Ping + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(2, true), + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(2, true), - // Stop backup - HRNPQ_MACRO_STOP_BACKUP_96(1, lsnStopStr, walSegmentStop, false), + // Stop backup + HRNPQ_MACRO_STOP_BACKUP_96(1, lsnStopStr, walSegmentStop, false), - // Get stop time - HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000), + // Get stop time + HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000), - HRNPQ_MACRO_DONE() - }); + HRNPQ_MACRO_DONE() + }); + } } // ----------------------------------------------------------------------------------------------------------------------------- else if (pgVersion == PG_VERSION_11) { ASSERT(!param.backupStandby); + ASSERT(!param.noArchiveCheck); if (param.errorAfterStart) { @@ -390,6 +491,7 @@ // Start backup HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_GE_10(1, walSegmentPrior), HRNPQ_MACRO_START_BACKUP_GE_10(1, param.startFast, lsnStartStr, walSegmentStart), HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), HRNPQ_MACRO_TABLESPACE_LIST_1(1, 32768, "tblspc32768"), @@ -413,7 +515,14 @@ // Start backup HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_GE_10(1, walSegmentStart), HRNPQ_MACRO_START_BACKUP_GE_10(1, param.startFast, lsnStartStr, walSegmentStart), + + // Switch WAL segment so it can be checked + HRNPQ_MACRO_CREATE_RESTORE_POINT(1, "X/X"), + HRNPQ_MACRO_WAL_SWITCH(1, "wal", walSegmentStart), + + // Get database and tablespace list HRNPQ_MACRO_DATABASE_LIST_1(1, "test1"), HRNPQ_MACRO_TABLESPACE_LIST_1(1, 32768, "tblspc32768"), @@ -421,8 +530,12 @@ HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 999), HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 1000), + // Ping + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + // Stop backup - HRNPQ_MACRO_STOP_BACKUP_GE_10(1, lsnStopStr, walSegmentStop, false), + HRNPQ_MACRO_STOP_BACKUP_GE_10(1, lsnStopStr, walSegmentStop, true), // Get stop time HRNPQ_MACRO_TIME_QUERY(1, (int64_t)backupTimeStart * 1000 + 2000), @@ -552,23 +665,6 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("pgFileSize, ignoreMissing=false, backupLabel, pgFileChecksumPage, pgFileChecksumPageLsnLimit"); - VariantList *paramList = varLstNew(); - varLstAdd(paramList, varNewStr(pgFile)); // pgFile - varLstAdd(paramList, varNewBool(false)); // pgFileIgnoreMissing - varLstAdd(paramList, varNewUInt64(8)); // pgFileSize - varLstAdd(paramList, varNewBool(false)); // pgFileCopyExactSize - varLstAdd(paramList, NULL); // pgFileChecksum - varLstAdd(paramList, varNewBool(true)); // pgFileChecksumPage - varLstAdd(paramList, varNewUInt64(0xFFFFFFFFFFFFFFFF)); // pgFileChecksumPageLsnLimit - varLstAdd(paramList, varNewStr(pgFile)); // repoFile - varLstAdd(paramList, varNewBool(false)); // repoFileHasReference - varLstAdd(paramList, varNewUInt(compressTypeNone)); // repoFileCompress - varLstAdd(paramList, varNewInt(1)); // repoFileCompressLevel - varLstAdd(paramList, varNewStr(backupLabel)); // backupLabel - varLstAdd(paramList, varNewBool(false)); // delta - varLstAdd(paramList, varNewUInt64(cipherTypeNone)); // cipherType - varLstAdd(paramList, NULL); // cipherSubPass - TEST_ASSIGN( result, backupFile( @@ -923,16 +1019,14 @@ HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_ERROR( - backupInit(infoBackupNew(PG_VERSION_91, 1000000000000000910, hrnPgCatalogVersion(PG_VERSION_91), NULL)), ConfigError, - "option 'backup-standby' not valid for PostgreSQL < 9.2"); + backupInit(infoBackupNew(PG_VERSION_91, HRN_PG_SYSTEMID_91, hrnPgCatalogVersion(PG_VERSION_91), NULL)), + ConfigError, "option 'backup-standby' not valid for PostgreSQL < 9.2"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("warn and reset when backup from standby used in offline mode"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 1000000000000000920})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_92); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); @@ -944,7 +1038,7 @@ HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_RESULT_VOID( - backupInit(infoBackupNew(PG_VERSION_92, 1000000000000000920, hrnPgCatalogVersion(PG_VERSION_92), NULL)), + backupInit(infoBackupNew(PG_VERSION_92, HRN_PG_SYSTEMID_92, hrnPgCatalogVersion(PG_VERSION_92), NULL)), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptBackupStandby), false, "check backup-standby"); @@ -955,9 +1049,7 @@ TEST_TITLE("error when pg_control does not match stanza"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 1000000000000001000})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_10); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); @@ -968,23 +1060,23 @@ HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_ERROR( - backupInit(infoBackupNew(PG_VERSION_11, 1000000000000001100, hrnPgCatalogVersion(PG_VERSION_11), NULL)), + backupInit(infoBackupNew(PG_VERSION_11, HRN_PG_SYSTEMID_11, hrnPgCatalogVersion(PG_VERSION_11), NULL)), BackupMismatchError, - "PostgreSQL version 10, system-id 1000000000000001000 do not match stanza version 11, system-id 1000000000000001100\n" + "PostgreSQL version 10, system-id " HRN_PG_SYSTEMID_10_Z " do not match stanza version 11, system-id" + " " HRN_PG_SYSTEMID_11_Z "\n" "HINT: is this the correct stanza?"); TEST_ERROR( - backupInit(infoBackupNew(PG_VERSION_10, 1000000000000001100, hrnPgCatalogVersion(PG_VERSION_10), NULL)), + backupInit(infoBackupNew(PG_VERSION_10, HRN_PG_SYSTEMID_11, hrnPgCatalogVersion(PG_VERSION_10), NULL)), BackupMismatchError, - "PostgreSQL version 10, system-id 1000000000000001000 do not match stanza version 10, system-id 1000000000000001100\n" + "PostgreSQL version 10, system-id " HRN_PG_SYSTEMID_10_Z " do not match stanza version 10, system-id" + " " HRN_PG_SYSTEMID_11_Z "\n" "HINT: is this the correct stanza?"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("reset start-fast when PostgreSQL < 8.4"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_83, .systemId = 1000000000000000830})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_83); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); @@ -996,7 +1088,7 @@ HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_RESULT_VOID( - backupInit(infoBackupNew(PG_VERSION_83, 1000000000000000830, hrnPgCatalogVersion(PG_VERSION_83), NULL)), + backupInit(infoBackupNew(PG_VERSION_83, HRN_PG_SYSTEMID_83, hrnPgCatalogVersion(PG_VERSION_83), NULL)), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptStartFast), false, "check start-fast"); @@ -1006,9 +1098,7 @@ TEST_TITLE("reset stop-auto when PostgreSQL < 9.3"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_84, .systemId = 1000000000000000840})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_84); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); @@ -1020,7 +1110,7 @@ HRN_CFG_LOAD(cfgCmdBackup, argList); TEST_RESULT_VOID( - backupInit(infoBackupNew(PG_VERSION_84, 1000000000000000840, hrnPgCatalogVersion(PG_VERSION_84), NULL)), + backupInit(infoBackupNew(PG_VERSION_84, HRN_PG_SYSTEMID_84, hrnPgCatalogVersion(PG_VERSION_84), NULL)), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptStopAuto), false, "check stop-auto"); @@ -1030,9 +1120,18 @@ TEST_TITLE("reset checksum-page when the cluster does not have checksums enabled"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_93); + + // Create stanza + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1"); + hrnCfgArgRawBool(argList, cfgOptOnline, false); + HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); + + cmdStanzaCreate(); + TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1"); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); @@ -1051,7 +1150,8 @@ }); TEST_RESULT_VOID( - dbFree(backupInit(infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary), + dbFree( + backupInit(infoBackupNew(PG_VERSION_93, HRN_PG_SYSTEMID_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page"); @@ -1062,9 +1162,7 @@ TEST_TITLE("ok if cluster checksums are enabled and checksum-page is any value"); // Create pg_control with page checksums - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93, .pageChecksum = true})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_93, .pageChecksum = true); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); @@ -1083,14 +1181,13 @@ }); TEST_RESULT_VOID( - dbFree(backupInit(infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary), + dbFree( + backupInit(infoBackupNew(PG_VERSION_93, HRN_PG_SYSTEMID_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page"); // Create pg_control without page checksums - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_93); harnessPqScriptSet((HarnessPq []) { @@ -1101,7 +1198,8 @@ }); TEST_RESULT_VOID( - dbFree(backupInit(infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary), + dbFree( + backupInit(infoBackupNew(PG_VERSION_93, HRN_PG_SYSTEMID_93, hrnPgCatalogVersion(PG_VERSION_93), NULL))->dbPrimary), "backup init"); TEST_RESULT_BOOL(cfgOptionBool(cfgOptChecksumPage), false, "check checksum-page"); } @@ -1112,18 +1210,27 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("sleep retries and stall error"); + // Create pg_control + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_93); + + // Create stanza StringList *argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1"); + hrnCfgArgRawBool(argList, cfgOptOnline, false); + HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); + + cmdStanzaCreate(); + TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg1"); hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1"); HRN_CFG_LOAD(cfgCmdBackup, argList); - // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .systemId = PG_VERSION_93})); - harnessPqScriptSet((HarnessPq []) { // Connect to primary @@ -1144,7 +1251,7 @@ }); BackupData *backupData = backupInit( - infoBackupNew(PG_VERSION_93, PG_VERSION_93, hrnPgCatalogVersion(PG_VERSION_93), NULL)); + infoBackupNew(PG_VERSION_93, HRN_PG_SYSTEMID_93, hrnPgCatalogVersion(PG_VERSION_93), NULL)); TEST_RESULT_INT(backupTime(backupData, true), 1575392588, "multiple tries for sleep"); TEST_ERROR(backupTime(backupData, true), KernelError, "PostgreSQL clock has not advanced to the next second after 3 tries"); @@ -1322,16 +1429,16 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("report job error"); - ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(strIdFromZ(stringIdBit5, "x"))); + ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(strIdFromZ("x"))); protocolParallelJobErrorSet(job, errorTypeCode(&AssertError), STRDEF("error message")); - TEST_ERROR(backupJobResult((Manifest *)1, NULL, STRDEF("log"), strLstNew(), job, 0, 0), AssertError, "error message"); + TEST_ERROR(backupJobResult((Manifest *)1, NULL, STRDEF("log"), strLstNew(), job, 0, NULL), AssertError, "error message"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("report host/100% progress on noop result"); // Create job that skips file - job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x"))); + job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ("x"))); PackWrite *const resultPack = protocolPackNew(); pckWriteU32P(resultPack, backupCopyResultNoOp); @@ -1353,8 +1460,10 @@ } OBJ_NEW_END(); - TEST_RESULT_UINT( - backupJobResult(manifest, STRDEF("host"), STRDEF("log-test"), strLstNew(), job, 0, 0), 0, "log noop result"); + uint64_t sizeProgress = 0; + + TEST_RESULT_VOID( + backupJobResult(manifest, STRDEF("host"), STRDEF("log-test"), strLstNew(), job, 0, &sizeProgress), "log noop result"); TEST_RESULT_LOG("P00 DETAIL: match file from prior backup host:log-test (0B, 100%)"); } @@ -1380,9 +1489,7 @@ HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_84, .systemId = 1000000000000000840})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_84); cmdStanzaCreate(); TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1"); @@ -1434,8 +1541,8 @@ "P00 INFO: new backup label = [FULL-1]\n" "P00 INFO: full backup size = 8KB, file total = 2", TEST_64BIT() ? - (TEST_BIG_ENDIAN() ? "749acedef8f8d5fe35fc20c0375657f876ccc38e" : "21e2ddc99cdf4cfca272eee4f38891146092e358") : - "8bb70506d988a8698d9e8cf90736ada23634571b"); + (TEST_BIG_ENDIAN() ? "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" : "6c1435a9f3c24a020794f58945ada456cb1d3bbe") : + "d432aca683e0443e97cf0600ba3a5a9efd7586fd"); // Make pg no longer appear to be running HRN_STORAGE_REMOVE(storagePgWrite(), PG_FILE_POSTMTRPID, .errorOnMissing = true); @@ -1631,10 +1738,7 @@ HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_95, .systemId = 1000000000000000950}), - .timeModified = backupTimeStart); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_95); cmdStanzaCreate(); TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1"); @@ -1683,7 +1787,7 @@ strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strZ(resumeLabel))))); // Run backup - testBackupPqScriptP(PG_VERSION_95, backupTimeStart); + testBackupPqScriptP(PG_VERSION_95, backupTimeStart, .noArchiveCheck = true, .noWal = true); TEST_RESULT_VOID(cmdBackup(), "backup"); TEST_RESULT_LOG( @@ -1828,6 +1932,7 @@ TEST_RESULT_LOG( "P00 INFO: execute exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000000105D95D3000000000, lsn = 5d95d30/0\n" + "P00 INFO: check archive for prior segment 0000000105D95D2F000000FF\n" "P00 WARN: resumable backup 20191003-105320F of same type exists -- remove invalid files and resume\n" "P00 DETAIL: remove path '" TEST_PATH "/repo/backup/test1/20191003-105320F/pg_data/bogus_path' from resumed" " backup\n" @@ -1880,7 +1985,7 @@ "[target:file]\n" "pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"size\":3" ",\"timestamp\":1570000000}\n" - "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570000000}\n" + "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570100000}\n" "pg_data/not-in-resume={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4" ",\"timestamp\":1570100000}\n" "pg_data/pg_xlog/0000000105D95D3000000000={\"size\":16777216,\"timestamp\":1570100002}\n" @@ -1991,6 +2096,7 @@ "P00 WARN: diff backup cannot alter compress-type option to 'none', reset to value in 20191003-105320F\n" "P00 INFO: execute exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000000105D9759000000000, lsn = 5d97590/0\n" + "P00 INFO: check archive for prior segment 0000000105D9758F000000FF\n" "P00 WARN: file 'time-mismatch2' has timestamp in the future, enabling delta checksum\n" "P00 WARN: resumable backup 20191003-105320F_20191004-144000D of same type exists" " -- remove invalid files and resume\n" @@ -2000,7 +2106,7 @@ " from resumed backup\n" "P00 DETAIL: remove file '" TEST_PATH "/repo/backup/test1/20191003-105320F_20191004-144000D/pg_data/resume-ref.gz'" " from resumed backup (reference in resumed manifest)\n" - "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n" "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n" "P00 WARN: resumed backup file pg_data/time-mismatch2 does not have expected checksum" " 984816fd329622876e14907634264e6f332e9fb3. The file will be recopied and backup will continue but this may be" @@ -2010,7 +2116,6 @@ "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/PG_VERSION (3B, [PCT]) checksum [SHA1]\n" "P01 DETAIL: backup file " TEST_PATH "/pg1/resume-ref (0B, [PCT])\n" "P00 DETAIL: hardlink pg_data/PG_VERSION to 20191003-105320F\n" - "P00 DETAIL: hardlink pg_data/global/pg_control to 20191003-105320F\n" "P00 DETAIL: hardlink pg_data/postgresql.conf to 20191003-105320F\n" "P00 INFO: execute exclusive pg_stop_backup() and wait for all WAL segments to archive\n" "P00 INFO: backup stop archive = 0000000105D9759000000000, lsn = 5d97590/800000\n" @@ -2037,7 +2142,7 @@ "[target:file]\n" "pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"reference\":\"20191003-105320F\"" ",\"size\":3,\"timestamp\":1570000000}\n" - "pg_data/global/pg_control={\"reference\":\"20191003-105320F\",\"size\":8192,\"timestamp\":1570000000}\n" + "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570200000}\n" "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\"" ",\"reference\":\"20191003-105320F\",\"size\":11,\"timestamp\":1570000000}\n" "pg_data/resume-ref={\"size\":0,\"timestamp\":1570200000}\n" @@ -2062,10 +2167,7 @@ { // Update pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 1000000000000000960}), - .timeModified = backupTimeStart); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_96); // Update version HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_96_STR, .timeModified = backupTimeStart); @@ -2095,6 +2197,9 @@ hrnCfgArgRawBool(argList, cfgOptArchiveCopy, true); HRN_CFG_LOAD(cfgCmdBackup, argList); + // Add pg_control to standby + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_96); + // Create file to copy from the standby. This file will be zero-length on the primary and non-zero-length on the standby // but no bytes will be copied. HRN_STORAGE_PUT_EMPTY(storagePgIdxWrite(0), PG_PATH_BASE "/1/1", .timeModified = backupTimeStart); @@ -2118,8 +2223,19 @@ // Set log level to warn because the following test uses multiple processes so the log order will not be deterministic harnessLogLevelSet(logLevelWarn); + // Run backup but error on first archive check + testBackupPqScriptP( + PG_VERSION_96, backupTimeStart, .noPriorWal = true, .backupStandby = true, .walCompressType = compressTypeGz); + TEST_ERROR( + cmdBackup(), ArchiveTimeoutError, + "WAL segment 0000000105DA69BF000000FF was not archived before the 100ms timeout\n" + "HINT: check the archive_command to ensure that all options are correct (especially --stanza).\n" + "HINT: check the PostgreSQL server log for errors.\n" + "HINT: run the 'start' command if the stanza was previously stopped."); + // Run backup but error on archive check - testBackupPqScriptP(PG_VERSION_96, backupTimeStart, .noWal = true, .backupStandby = true); + testBackupPqScriptP( + PG_VERSION_96, backupTimeStart, .noWal = true, .backupStandby = true, .walCompressType = compressTypeGz); TEST_ERROR( cmdBackup(), ArchiveTimeoutError, "WAL segment 0000000105DA69C000000000 was not archived before the 100ms timeout\n" @@ -2195,13 +2311,7 @@ { // Update pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer( - (PgControl){ - .version = PG_VERSION_11, .systemId = 1000000000000001100, .pageChecksum = true, - .walSegmentSize = 1024 * 1024}), - .timeModified = backupTimeStart); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_11, .pageChecksum = true, .walSegmentSize = 1024 * 1024); // Update version HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_11_STR, .timeModified = backupTimeStart); @@ -2307,6 +2417,7 @@ TEST_RESULT_LOG( "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000000105DB5DE000000000, lsn = 5db5de0/0\n" + "P00 INFO: check archive for segment 0000000105DB5DE000000000\n" "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/3 (32KB, [PCT]) checksum [SHA1]\n" "P00 WARN: invalid page checksums found in file " TEST_PATH "/pg1/base/1/3 at pages 0, 2-3\n" "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/4 (24KB, [PCT]) checksum [SHA1]\n" @@ -2322,12 +2433,13 @@ "P00 INFO: execute non-exclusive pg_stop_backup() and wait for all WAL segments to archive\n" "P00 INFO: backup stop archive = 0000000105DB5DE000000002, lsn = 5db5de0/280000\n" "P00 DETAIL: wrote 'backup_label' file returned from pg_stop_backup()\n" + "P00 DETAIL: wrote 'tablespace_map' file returned from pg_stop_backup()\n" "P00 INFO: check archive for segment(s) 0000000105DB5DE000000000:0000000105DB5DE000000002\n" "P00 DETAIL: copy segment 0000000105DB5DE000000000 to backup\n" "P00 DETAIL: copy segment 0000000105DB5DE000000001 to backup\n" "P00 DETAIL: copy segment 0000000105DB5DE000000002 to backup\n" "P00 INFO: new backup label = 20191027-181320F\n" - "P00 INFO: full backup size = [SIZE], file total = 12"); + "P00 INFO: full backup size = [SIZE], file total = 13"); TEST_RESULT_STR( testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/20191027-181320F")), @@ -2349,6 +2461,7 @@ "pg_data/pg_wal/0000000105DB5DE000000001.gz {file, s=1048576}\n" "pg_data/pg_wal/0000000105DB5DE000000002.gz {file, s=1048576}\n" "pg_data/postgresql.conf.gz {file, s=11}\n" + "pg_data/tablespace_map.gz {file, s=19}\n" "pg_tblspc {path}\n" "pg_tblspc/32768 {path}\n" "pg_tblspc/32768/PG_11_201809051 {path}\n" @@ -2379,6 +2492,8 @@ "pg_data/pg_wal/0000000105DB5DE000000002={\"size\":1048576,\"timestamp\":1572200002}\n" "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11" ",\"timestamp\":1570000000}\n" + "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19" + ",\"timestamp\":1572200002}\n" "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"mas""ter\":false,\"size\":0" ",\"timestamp\":1572200000}\n" "\n" @@ -2408,8 +2523,6 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error when pg_control not present"); - backupTimeStart = BACKUP_EPOCH + 2300000; - { // Load options StringList *argList = strLstNew(); @@ -2421,8 +2534,11 @@ hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true); HRN_CFG_LOAD(cfgCmdBackup, argList); + // Preserve prior timestamp on pg_control + testBackupPqScriptP(PG_VERSION_11, BACKUP_EPOCH + 2300000, .errorAfterStart = true); + HRN_PG_CONTROL_TIME(storagePg(), backupTimeStart); + // Run backup - testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .errorAfterStart = true); TEST_ERROR( cmdBackup(), FileMissingError, "pg_control must be present in all online backups\n" @@ -2432,7 +2548,8 @@ TEST_RESULT_LOG( "P00 INFO: last backup label = 20191027-181320F, version = " PROJECT_VERSION "\n" "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" - "P00 INFO: backup start archive = 0000000105DB764000000000, lsn = 5db7640/0"); + "P00 INFO: backup start archive = 0000000105DB764000000000, lsn = 5db7640/0\n" + "P00 INFO: check archive for prior segment 0000000105DB763F00000FFF"); // Remove partial backup so it won't be resumed (since it errored before any checksums were written) HRN_STORAGE_PATH_REMOVE(storageRepoWrite(), STORAGE_REPO_BACKUP "/20191027-181320F_20191028-220000I", .recurse = true); @@ -2459,34 +2576,32 @@ hrnCfgArgRawBool(argList, cfgOptRepoHardlink, true); HRN_CFG_LOAD(cfgCmdBackup, argList); - // Update pg_control timestamp - HRN_STORAGE_TIME(storagePg(), "global/pg_control", backupTimeStart); - // Run backup. Make sure that the timeline selected converts to hexdecimal that can't be interpreted as decimal. - testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .timeline = 0x2C); + testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .timeline = 0x2C, .walTotal = 2); TEST_RESULT_VOID(cmdBackup(), "backup"); TEST_RESULT_LOG( "P00 INFO: last backup label = 20191027-181320F, version = " PROJECT_VERSION "\n" "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000002C05DB8EB000000000, lsn = 5db8eb0/0\n" + "P00 INFO: check archive for segment 0000002C05DB8EB000000000\n" "P00 WARN: a timeline switch has occurred since the 20191027-181320F backup, enabling delta checksum\n" " HINT: this is normal after restoring from backup or promoting a standby.\n" - "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (8KB, [PCT]) checksum [SHA1]\n" "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/base/1/1 (8KB, [PCT]) checksum [SHA1]\n" "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/postgresql.conf (11B, [PCT]) checksum [SHA1]\n" "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/PG_VERSION (2B, [PCT]) checksum [SHA1]\n" "P00 DETAIL: hardlink pg_data/PG_VERSION to 20191027-181320F\n" "P00 DETAIL: hardlink pg_data/base/1/1 to 20191027-181320F\n" - "P00 DETAIL: hardlink pg_data/global/pg_control to 20191027-181320F\n" "P00 DETAIL: hardlink pg_data/postgresql.conf to 20191027-181320F\n" "P00 DETAIL: hardlink pg_tblspc/32768/PG_11_201809051/1/5 to 20191027-181320F\n" "P00 INFO: execute non-exclusive pg_stop_backup() and wait for all WAL segments to archive\n" - "P00 INFO: backup stop archive = 0000002C05DB8EB000000000, lsn = 5db8eb0/80000\n" + "P00 INFO: backup stop archive = 0000002C05DB8EB000000001, lsn = 5db8eb0/180000\n" "P00 DETAIL: wrote 'backup_label' file returned from pg_stop_backup()\n" - "P00 INFO: check archive for segment(s) 0000002C05DB8EB000000000:0000002C05DB8EB000000000\n" + "P00 DETAIL: wrote 'tablespace_map' file returned from pg_stop_backup()\n" + "P00 INFO: check archive for segment(s) 0000002C05DB8EB000000000:0000002C05DB8EB000000001\n" "P00 INFO: new backup label = 20191027-181320F_20191030-014640I\n" - "P00 INFO: incr backup size = [SIZE], file total = 6"); + "P00 INFO: incr backup size = [SIZE], file total = 7"); TEST_RESULT_STR_Z( testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")), @@ -2503,6 +2618,7 @@ "pg_data/pg_tblspc/32768 {link, d=../../pg_tblspc/32768}\n" "pg_data/pg_wal {path}\n" "pg_data/postgresql.conf.gz {file, s=11}\n" + "pg_data/tablespace_map.gz {file, s=19}\n" "pg_tblspc {path}\n" "pg_tblspc/32768 {path}\n" "pg_tblspc/32768/PG_11_201809051 {path}\n" @@ -2521,9 +2637,11 @@ ",\"timestamp\":1572400002}\n" "pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true" ",\"mas""ter\":false,\"reference\":\"20191027-181320F\",\"size\":8192,\"timestamp\":1572200000}\n" - "pg_data/global/pg_control={\"reference\":\"20191027-181320F\",\"size\":8192,\"timestamp\":1572400000}\n" + "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572400000}\n" "pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\"" ",\"reference\":\"20191027-181320F\",\"size\":11,\"timestamp\":1570000000}\n" + "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19" + ",\"timestamp\":1572400002}\n" "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"mas""ter\":false,\"reference\":\"20191027-181320F\"" ",\"size\":0,\"timestamp\":1572200000}\n" "\n" diff -Nru pgbackrest-2.36/test/src/module/command/checkTest.c pgbackrest-2.37/test/src/module/command/checkTest.c --- pgbackrest-2.36/test/src/module/command/checkTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/checkTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -56,6 +56,8 @@ //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("standby only, repo local - fail to find primary database"); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_92); + harnessPqScriptSet((HarnessPq []) { HRNPQ_MACRO_OPEN_GE_92(1, "dbname='postgres' port=5432", PG_VERSION_92, TEST_PATH "/pg", true, NULL, NULL), @@ -71,13 +73,15 @@ argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg"); - hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 8, "/path/to/standby2"); + hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 8, TEST_PATH "/pg8"); hrnCfgArgKeyRawZ(argList, cfgOptPgPort, 8, "5433"); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); hrnCfgArgKeyRawZ(argList, cfgOptRepoHost, 2, "repo.domain.com"); hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, ".5"); HRN_CFG_LOAD(cfgCmdCheck, argList); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_92); + // Two standbys found but no primary harnessPqScriptSet((HarnessPq []) { @@ -112,7 +116,10 @@ // Only confirming we get passed the check for repoIsLocal || more than one pg-path configured TEST_ERROR( - cmdCheck(), FileMissingError, "unable to open missing file '" TEST_PATH "/pg/global/pg_control' for read"); + cmdCheck(), DbMismatchError, + "version '9.2' and path '/pgdata' queried from cluster do not match version '9.2' and '" TEST_PATH "/pg' read from" + " '" TEST_PATH "/pg/global/pg_control'\n" + "HINT: the pg1-path and pg1-port settings likely reference different clusters."); //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("backup-standby set without standby"); @@ -134,8 +141,18 @@ }); TEST_ERROR( - cmdCheck(), FileMissingError, "unable to open missing file '" TEST_PATH "/pg/global/pg_control' for read"); - TEST_RESULT_LOG("P00 WARN: option 'backup-standby' is enabled but standby is not properly configured"); + cmdCheck(), FileMissingError, + "unable to load info file '" TEST_PATH "/repo/archive/test1/archive.info' or '" TEST_PATH + "/repo/archive/test1/archive.info.copy':\n" + "FileMissingError: unable to open missing file '" TEST_PATH "/repo/archive/test1/archive.info' for read\n" + "FileMissingError: unable to open missing file '" TEST_PATH "/repo/archive/test1/archive.info.copy' for read\n" + "HINT: archive.info cannot be opened but is required to push/get WAL segments.\n" + "HINT: is archive_command configured correctly in postgresql.conf?\n" + "HINT: has a stanza-create been performed?\n" + "HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme."); + TEST_RESULT_LOG( + "P00 WARN: option 'backup-standby' is enabled but standby is not properly configured\n" + "P00 INFO: check repo1 configuration (primary)"); //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("standby and primary database - standby path doesn't match pg_control"); @@ -150,9 +167,7 @@ HRN_CFG_LOAD(cfgCmdCheck, argList); // Create pg_control for standby - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 6569239123849665679})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_92); // Standby database path doesn't match pg_control harnessPqScriptSet((HarnessPq []) @@ -176,31 +191,29 @@ TEST_TITLE("standby and primary database - error on primary but standby check ok"); // Create pg_control for primary - HRN_STORAGE_PUT( - storagePgIdxWrite(1), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 6569239123849665679})); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_92); // Create info files HRN_INFO_PUT( storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, "[db]\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_92_Z "\n" "db-version=\"9.2\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.2\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_92_Z ",\"db-version\":\"9.2\"}\n"); HRN_INFO_PUT( storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, "[db]\n" "db-catalog-version=201608131\n" "db-control-version=920\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_92_Z "\n" "db-version=\"9.2\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201608131,\"db-control-version\":920,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201608131,\"db-control-version\":920,\"db-system-id\":" HRN_PG_SYSTEMID_92_Z "," "\"db-version\":\"9.2\"}\n"); // Single repo config - error when checking archive mode setting on database @@ -272,22 +285,22 @@ storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE, "[db]\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_92_Z "\n" "db-version=\"9.2\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.2\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_92_Z ",\"db-version\":\"9.2\"}\n"); HRN_INFO_PUT( storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE, "[db]\n" "db-catalog-version=201608131\n" "db-control-version=920\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_92_Z "\n" "db-version=\"9.2\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201608131,\"db-control-version\":920,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201608131,\"db-control-version\":920,\"db-system-id\":" HRN_PG_SYSTEMID_92_Z "," "\"db-version\":\"9.2\"}\n"); // Error when WAL segment not found @@ -402,6 +415,9 @@ hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); HRN_CFG_LOAD(cfgCmdCheck, argList); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_92); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_92); + DbGetResult db = {0}; harnessPqScriptSet((HarnessPq []) @@ -572,9 +588,7 @@ HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); // Create pg_control - HRN_STORAGE_PUT( - storagePgIdxWrite(0), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 6569239123849665679})); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_96); // Create info files TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create - encryption"); @@ -583,7 +597,7 @@ // Version mismatch TEST_ERROR( checkStanzaInfoPg( - storageRepoIdx(0), PG_VERSION_94, 6569239123849665679, cfgOptionIdxStrId(cfgOptRepoCipherType, 0), + storageRepoIdx(0), PG_VERSION_94, HRN_PG_SYSTEMID_94, cfgOptionIdxStrId(cfgOptRepoCipherType, 0), cfgOptionIdxStr(cfgOptRepoCipherPass, 0)), FileInvalidError, "backup and archive info files exist but do not match the database\n" diff -Nru pgbackrest-2.36/test/src/module/command/helpTest.c pgbackrest-2.37/test/src/module/command/helpTest.c --- pgbackrest-2.36/test/src/module/command/helpTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/helpTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -62,6 +62,8 @@ " repo-get Get a file from a repository.\n" " repo-ls List files in a repository.\n" " restore Restore a database cluster.\n" + " server pgBackRest server.\n" + " server-ping Ping pgBackRest server.\n" " stanza-create Create the required stanza data.\n" " stanza-delete Delete a stanza.\n" " stanza-upgrade Upgrade a stanza.\n" @@ -120,17 +122,6 @@ } // ***************************************************************************************************************************** - if (testBegin("helpRenderValue()")) - { - TEST_RESULT_STR_Z(helpRenderValue(varNewBool(true), cfgOptTypeBoolean), "y", "boolean y"); - TEST_RESULT_STR_Z(helpRenderValue(varNewBool(false), cfgOptTypeBoolean), "n", "boolean n"); - TEST_RESULT_STR_Z(helpRenderValue(varNewStrZ("test-string"), cfgOptTypeString), "test-string", "string"); - TEST_RESULT_STR_Z(helpRenderValue(varNewInt64(1234), cfgOptTypeInteger), "1234", "int"); - TEST_RESULT_STR_Z(helpRenderValue(varNewInt64(1234000), cfgOptTypeTime), "1234", "time"); - TEST_RESULT_STR_Z(helpRenderValue(NULL, cfgOptTypeString), NULL, "null"); - } - - // ***************************************************************************************************************************** if (testBegin("helpRender()")) { StringList *argList = NULL; @@ -222,7 +213,9 @@ "General Options:\n" "\n" " --buffer-size buffer size for I/O operations\n" - " [current=32768, default=1048576]\n" + " [current=32768, default=1MiB]\n" + " --cmd pgBackRest command\n" + " [default=/path/to/pgbackrest]\n" " --cmd-ssh SSH client command [default=ssh]\n" " --compress-level-network network compression level [default=3]\n" " --config pgBackRest configuration file\n" @@ -276,6 +269,9 @@ " --repo-gcs-key-type GCS repository key type [default=service]\n" " --repo-host repository host when operating remotely via\n" " SSH [current=backup.example.net]\n" + " --repo-host-ca-file repository host certificate authority file\n" + " --repo-host-ca-path repository host certificate authority path\n" + " --repo-host-cert-file repository host certificate file\n" " --repo-host-cmd repository host pgBackRest command\n" " [default=/path/to/pgbackrest]\n" " --repo-host-config pgBackRest repository host configuration\n" @@ -285,7 +281,9 @@ " include path [default=/etc/pgbackrest/conf.d]\n" " --repo-host-config-path pgBackRest repository host configuration\n" " path [default=/etc/pgbackrest]\n" + " --repo-host-key-file repository host key file\n" " --repo-host-port repository host port when repo-host is set\n" + " --repo-host-type repository host protocol type [default=ssh]\n" " --repo-host-user repository host user when repo-host is set\n" " [default=pgbackrest]\n" " --repo-path path where backups and archive are stored\n" @@ -370,12 +368,13 @@ "of buffers used depends on options and each operation may use additional\n" "memory, e.g. gz compression may use an additional 256KiB of memory.\n" "\n" - "Size can be entered in bytes (default) or KB, MB, GB, TB, or PB where the\n" - "multiplier is a power of 1024. For example, the case-insensitive value 32k (or\n" - "32KB) can be used instead of 32768.\n" + "Size can be entered in bytes (default) or KiB, MiB, GiB, TiB, or PiB where the\n" + "multiplier is a power of 1024. For example, the case-insensitive value 5GiB (or\n" + "5GB, 5g) can be used instead of 5368709120. Fractional values such as 2.5GiB\n" + "are not allowed, use 2560MiB instead.\n" "\n" - "Allowed values, in bytes, are 16384, 32768, 65536, 131072, 262144, 524288,\n" - "1048576, 2097152, 4194304, 8388608, and 16777216.\n", + "Allowed values are 16KiB, 32KiB, 64KiB, 128KiB, 256KiB, 512KiB, 1MiB, 2MiB,\n" + "4MiB, 8MiB, and 16MiB.\n", helpVersion)); argList = strLstNew(); @@ -384,13 +383,13 @@ strLstAddZ(argList, "archive-push"); strLstAddZ(argList, "buffer-size"); TEST_RESULT_VOID(testCfgLoad(argList), "help for archive-push command, buffer-size option"); - TEST_RESULT_STR(helpRender(helpData), strNewFmt("%s\ndefault: 1048576\n", optionHelp), "check text"); + TEST_RESULT_STR(helpRender(helpData), strNewFmt("%s\ndefault: 1MiB\n", optionHelp), "check text"); // Set a current value - hrnCfgArgRawZ(argList, cfgOptBufferSize, "32768"); + hrnCfgArgRawZ(argList, cfgOptBufferSize, "32k"); TEST_RESULT_VOID(testCfgLoad(argList), "help for archive-push command, buffer-size option"); TEST_RESULT_STR( - helpRender(helpData), strNewFmt("%s\ncurrent: 32768\ndefault: 1048576\n", optionHelp), "check text, current value"); + helpRender(helpData), strNewFmt("%s\ncurrent: 32k\ndefault: 1MiB\n", optionHelp), "check text, current value"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("deprecated host option names"); diff -Nru pgbackrest-2.36/test/src/module/command/repoTest.c pgbackrest-2.37/test/src/module/command/repoTest.c --- pgbackrest-2.36/test/src/module/command/repoTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/repoTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -45,12 +45,12 @@ TEST_TITLE("missing directory"); Buffer *output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "missing directory (text)"); TEST_RESULT_STR_Z(strNewBuf(output), "", "check output"); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("json")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_JSON)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "missing directory (json)"); TEST_RESULT_STR_Z( strNewBuf(output), @@ -65,12 +65,12 @@ HRN_STORAGE_PATH_CREATE(storageRepoWrite(), NULL, .mode = 0700); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "empty directory (text)"); TEST_RESULT_STR_Z(strNewBuf(output), "", "check output"); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("json")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_JSON)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "empty directory (json)"); TEST_RESULT_STR_Z( strNewBuf(output), @@ -99,7 +99,7 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("add path and file"); - cfgOptionSet(cfgOptSort, cfgSourceParam, VARSTRDEF("asc")); + cfgOptionSet(cfgOptSort, cfgSourceParam, VARUINT64(CFGOPTVAL_SORT_ASC)); HRN_STORAGE_PATH_CREATE(storageRepoWrite(), "bbb"); HRN_STORAGE_PUT_Z(storageRepoWrite(), "aaa", "TESTDATA", .timeModified = 1578671569); @@ -109,12 +109,12 @@ HRN_SYSTEM("mkfifo " TEST_PATH "/repo/pipe"); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "path and file (text)"); TEST_RESULT_STR_Z(strNewBuf(output), "aaa\nbbb\nlink\npipe\n", "check output"); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("json")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_JSON)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "path and file (json)"); TEST_RESULT_STR_Z( strNewBuf(output), @@ -130,10 +130,10 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("reverse sort"); - cfgOptionSet(cfgOptSort, cfgSourceParam, VARSTRDEF("desc")); + cfgOptionSet(cfgOptSort, cfgSourceParam, VARUINT64(CFGOPTVAL_SORT_DESC)); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "path and file (text)"); TEST_RESULT_STR_Z(strNewBuf(output), "pipe\nlink\nbbb\naaa\n", "check output"); @@ -143,7 +143,7 @@ cfgOptionSet(cfgOptRecurse, cfgSourceParam, VARBOOL(true)); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "filter"); TEST_RESULT_STR_Z(strNewBuf(output), "pipe\nlink\nbbb/ccc\nbbb\naaa\n", "check output"); @@ -153,7 +153,7 @@ cfgOptionSet(cfgOptFilter, cfgSourceParam, VARSTRDEF("^aaa$")); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "filter"); TEST_RESULT_STR_Z(strNewBuf(output), "aaa\n", "check output"); @@ -165,7 +165,7 @@ HRN_CFG_LOAD(cfgCmdRepoLs, argListTmp); output = bufNew(0); - cfgOptionSet(cfgOptOutput, cfgSourceParam, VARSTRDEF("text")); + cfgOptionSet(cfgOptOutput, cfgSourceParam, VARUINT64(CFGOPTVAL_OUTPUT_TEXT)); TEST_RESULT_VOID(storageListRender(ioBufferWriteNew(output)), "subdirectory"); TEST_RESULT_STR_Z(strNewBuf(output), "ccc\n", "check output"); diff -Nru pgbackrest-2.36/test/src/module/command/restoreTest.c pgbackrest-2.37/test/src/module/command/restoreTest.c --- pgbackrest-2.36/test/src/module/command/restoreTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/restoreTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -434,12 +434,11 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("invalid target time format"); - TEST_RESULT_INT(getEpoch(STRDEF("Tue, 15 Nov 1994 12:45:26")), 0, "invalid date time format"); - TEST_RESULT_LOG( - "P00 WARN: automatic backup set selection cannot be performed with provided time 'Tue, 15 Nov 1994 12:45:26'," - " latest backup set will be used\n" - " HINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone" - " (+/- HH or HHMM or HH:MM) - if timezone is omitted, local time is assumed (for UTC use +00)"); + TEST_ERROR( + getEpoch(STRDEF("Tue, 15 Nov 1994 12:45:26")), FormatError, + "automatic backup set selection cannot be performed with provided time 'Tue, 15 Nov 1994 12:45:26'\n" + "HINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone (+/- HH or HHMM or HH:MM) - if" + " timezone is omitted, local time is assumed (for UTC use +00)"); setenv("TZ", "UTC", true); } @@ -522,7 +521,7 @@ TEST_RESULT_UINT(backupData.repoIdx, 0, "backup set found, repo1"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("target time, multi repo, latest used"); + TEST_TITLE("target time, multi repo"); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza ,"test1"); @@ -551,23 +550,17 @@ "\n" TEST_RESTORE_BACKUP_INFO_DB); - TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set"); - TEST_RESULT_STR_Z(backupData.backupSet, "20161219-212741F_20161219-212918I", "default to latest backup set"); - TEST_RESULT_UINT(backupData.repoIdx, 0, "repo1 chosen because of priority order"); - TEST_RESULT_LOG( - "P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', repo1: latest backup set" - " will be used"); + TEST_ERROR( + restoreBackupSet(), BackupSetInvalidError, + "unable to find backup set with stop time less than '2016-12-19 16:27:30-0500'"); // Request repo2 - latest from repo2 will be chosen hrnCfgArgRawZ(argList, cfgOptRepo, "2"); HRN_CFG_LOAD(cfgCmdRestore, argList); - TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set"); - TEST_RESULT_STR_Z(backupData.backupSet, "20201212-201243F", "default to latest backup set"); - TEST_RESULT_UINT(backupData.repoIdx, 1, "repo2 chosen because repo option set"); - TEST_RESULT_LOG( - "P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', repo2: latest backup set" - " will be used"); + TEST_ERROR( + restoreBackupSet(), BackupSetInvalidError, + "unable to find backup set with stop time less than '2016-12-19 16:27:30-0500'"); // Switch paths so newest on repo1 argList = strLstNew(); @@ -580,12 +573,9 @@ HRN_CFG_LOAD(cfgCmdRestore, argList); - TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set"); - TEST_RESULT_STR_Z(backupData.backupSet, "20201212-201243F", "default to latest backup set"); - TEST_RESULT_UINT(backupData.repoIdx, 0, "repo1 chosen because of priority order"); - TEST_RESULT_LOG( - "P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', repo1: latest backup set" - " will be used"); + TEST_ERROR( + restoreBackupSet(), BackupSetInvalidError, + "unable to find backup set with stop time less than '2016-12-19 16:27:30-0500'"); argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza ,"test1"); @@ -597,14 +587,11 @@ HRN_CFG_LOAD(cfgCmdRestore, argList); - TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set"); - TEST_RESULT_STR_Z(backupData.backupSet, "20161219-212741F_20161219-212918I", "time invalid format, default latest"); - TEST_RESULT_UINT(backupData.repoIdx, 0, "repo1 chosen because of priority order"); - TEST_RESULT_LOG( - "P00 WARN: automatic backup set selection cannot be performed with provided time 'Tue, 15 Nov 1994 12:45:26'," - " latest backup set will be used\n" - " HINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone" - " (+/- HH or HHMM or HH:MM) - if timezone is omitted, local time is assumed (for UTC use +00)"); + TEST_ERROR( + restoreBackupSet(), FormatError, + "automatic backup set selection cannot be performed with provided time 'Tue, 15 Nov 1994 12:45:26'\n" + "HINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone (+/- HH or HHMM or HH:MM) - if" + " timezone is omitted, local time is assumed (for UTC use +00)"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("target time, multi repo, no candidates found"); @@ -623,7 +610,9 @@ HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO_DB); HRN_INFO_PUT(storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO_DB); - TEST_ERROR(restoreBackupSet(), BackupSetInvalidError, "no backup set found to restore"); + TEST_ERROR( + restoreBackupSet(), BackupSetInvalidError, + "unable to find backup set with stop time less than '2016-12-19 16:27:30-0500'"); TEST_RESULT_LOG( "P00 WARN: repo1: [BackupSetInvalidError] no backup sets to restore\n" "P00 WARN: repo2: [BackupSetInvalidError] no backup sets to restore"); @@ -1585,6 +1574,20 @@ "check recovery options"); // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("user-specified cmd"); + + argList = strLstDup(argBaseList); + hrnCfgArgRawZ(argList, cfgOptCmd, "/usr/local/bin/pg_wrapper.sh"); + HRN_CFG_LOAD(cfgCmdRestore, argList); + + TEST_RESULT_STR_Z( + restoreRecoveryConf(PG_VERSION_94, restoreLabel), + RECOVERY_SETTING_HEADER + "restore_command = '/usr/local/bin/pg_wrapper.sh --lock-path=" HRN_PATH "/lock --log-path=" HRN_PATH " --pg1-path=/pg" + " --repo1-path=/repo --stanza=test1 archive-get %f \"%p\"'\n", + "restore_command invokes /usr/local/bin/pg_wrapper.sh per --cmd option"); + + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("override restore_command"); hrnCfgArgRawZ(argBaseList, cfgOptRecoveryOption, "restore-command=my_restore_command"); @@ -2709,6 +2712,14 @@ // Write recovery.conf so we don't get a preserve warning HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_RECOVERYCONF, "Some Settings"); + // Covert pg_wal to a path so it will be removed + HRN_STORAGE_REMOVE(storagePgWrite(), "pg_wal"); + HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_wal"); + + // Covert pg_hba.conf to a path so it will be removed + HRN_STORAGE_REMOVE(storagePgWrite(), "pg_hba.conf"); + HRN_STORAGE_PUT_Z(storagePgWrite(), "pg_hba.conf", BOGUS_STR); + // Update the manifest with online = true to test recovery start time logging manifest->pub.data.backupOptionOnline = true; manifest->pub.data.backupTimestampStart = 1482182958; @@ -2737,10 +2748,14 @@ "P00 DETAIL: skip 'tablespace_map' -- tablespace links will be created based on mappings\n" "P00 DETAIL: remove 'global/pg_control' so cluster will not start if restore does not complete\n" "P00 INFO: remove invalid files/links/paths from '" TEST_PATH "/pg'\n" + "P00 DETAIL: remove invalid file '" TEST_PATH "/pg/pg_hba.conf'\n" + "P00 DETAIL: remove invalid path '" TEST_PATH "/pg/pg_wal'\n" "P00 DETAIL: remove invalid link '" TEST_PATH "/pg/pg_xact'\n" "P00 INFO: remove invalid files/links/paths from '" TEST_PATH "/wal'\n" "P00 INFO: remove invalid files/links/paths from '" TEST_PATH "/ts/1/PG_10_201707211'\n" + "P00 DETAIL: create symlink '" TEST_PATH "/pg/pg_wal' to '../wal'\n" "P00 DETAIL: create path '" TEST_PATH "/pg/pg_xact'\n" + "P00 DETAIL: create symlink '" TEST_PATH "/pg/pg_hba.conf' to '../config/pg_hba.conf'\n" "P01 DETAIL: restore zeroed file " TEST_PATH "/pg/base/32768/32769 (32KB, 49%)\n" "P01 DETAIL: restore file " TEST_PATH "/pg/base/16384/16385 - exists and matches backup (16KB, 74%)" " checksum d74e5f7ebe52a3ed468ba08c5b6aefaccd1ca88f\n" diff -Nru pgbackrest-2.36/test/src/module/command/serverTest.c pgbackrest-2.37/test/src/module/command/serverTest.c --- pgbackrest-2.36/test/src/module/command/serverTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/serverTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -1,6 +1,7 @@ /*********************************************************************************************************************************** Test Server Command ***********************************************************************************************************************************/ +#include "common/exit.h" #include "storage/posix/storage.h" #include "storage/remote/storage.h" @@ -26,7 +27,7 @@ { TEST_TITLE("server"); - HRN_FORK_BEGIN(.timeout = 5000) + HRN_FORK_BEGIN(.timeout = 15000) { HRN_FORK_CHILD_BEGIN(.prefix = "client repo") { @@ -109,27 +110,72 @@ } HRN_FORK_CHILD_END(); - HRN_FORK_PARENT_BEGIN(.prefix = "server") + HRN_FORK_PARENT_BEGIN(.prefix = "client control") { - StringList *argList = strLstNew(); - hrnCfgArgRawZ(argList, cfgOptTlsServerCaFile, HRN_SERVER_CA); - hrnCfgArgRawZ(argList, cfgOptTlsServerCertFile, HRN_SERVER_CERT); - hrnCfgArgRawZ(argList, cfgOptTlsServerKeyFile, HRN_SERVER_KEY); - hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "pgbackrest-client=db"); - hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0)); - HRN_CFG_LOAD(cfgCmdServerStart, argList); - - // Write a config file to demonstrate that settings are loaded - HRN_STORAGE_PUT_Z(storageTest, "pgbackrest.conf", "[global]\nrepo1-path=" TEST_PATH "/repo"); - - // Get pid of this process to identify child process later - pid_t pid = getpid(); - - TEST_RESULT_VOID(cmdServer(3), "server"); - - // If this is a child process then exit immediately - if (pid != getpid()) - exit(0); + HRN_FORK_BEGIN(.timeout = 15000) + { + HRN_FORK_CHILD_BEGIN(.prefix = "server") + { + // Write a config file to demonstrate that options are loaded and reloaded + HRN_STORAGE_PUT_Z( + storageTest, + "pgbackrest.conf", + "[global]\n" + CFGOPT_TLS_SERVER_CA_FILE "=" HRN_SERVER_CA "\n" + CFGOPT_TLS_SERVER_CERT_FILE "=" HRN_SERVER_CERT "\n" + CFGOPT_TLS_SERVER_KEY_FILE "=" HRN_SERVER_KEY "\n" + CFGOPT_TLS_SERVER_AUTH "=pgbackrest-client=db\n" + "repo1-path=" TEST_PATH "/repo\n"); + + StringList *argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptConfig, TEST_PATH "/pgbackrest.conf"); + hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0)); + HRN_CFG_LOAD(cfgCmdServer, argList); + + // Init exit signal handlers + exitInit(); + + // No log testing needed + harnessLogLevelSet(logLevelError); + + // Add a fake pid to ensure SIGTERM is sent to unterminated children + cmdServerInit(); + + int fakePid = INT_MAX; + lstAdd(serverLocal.processList, &fakePid); + + // Get pid of this process to identify child process later + pid_t pid = getpid(); + + // Add parameters to arg list required for a reload + strLstInsert(argList, 0, cfgExe()); + strLstAddZ(argList, CFGCMD_SERVER); + + TEST_RESULT_VOID(cmdServer(strLstSize(argList), strLstPtr(argList)), "server"); + + // If this is a child process then exit immediately + if (pid != getpid()) + { + HRN_FORK_CHILD_NOTIFY_PUT(); + exit(0); + } + } + HRN_FORK_CHILD_END(); + + HRN_FORK_PARENT_BEGIN(.prefix = "server control") + { + // Wait for forked server processes to exit + HRN_FORK_PARENT_NOTIFY_GET(0); + kill(HRN_FORK_PROCESS_ID(0), SIGHUP); + HRN_FORK_PARENT_NOTIFY_GET(0); + HRN_FORK_PARENT_NOTIFY_GET(0); + + // Send term to server processes + kill(HRN_FORK_PROCESS_ID(0), SIGTERM); + } + HRN_FORK_PARENT_END(); + } + HRN_FORK_END(); // Wait for both child processes to exit HRN_FORK_PARENT_NOTIFY_GET(0); @@ -157,7 +203,7 @@ TEST_ERROR(cmdServerPing(), ParamInvalidError, "extra parameters found"); - HRN_FORK_BEGIN(.timeout = 5000) + HRN_FORK_BEGIN(.timeout = 15000) { HRN_FORK_CHILD_BEGIN(.prefix = "client") @@ -185,24 +231,52 @@ } HRN_FORK_CHILD_END(); - HRN_FORK_PARENT_BEGIN(.prefix = "server") + HRN_FORK_PARENT_BEGIN(.prefix = "client control") { - StringList *argList = strLstNew(); - hrnCfgArgRawZ(argList, cfgOptTlsServerCaFile, HRN_SERVER_CA); - hrnCfgArgRawZ(argList, cfgOptTlsServerCertFile, HRN_SERVER_CERT); - hrnCfgArgRawZ(argList, cfgOptTlsServerKeyFile, HRN_SERVER_KEY); - hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "bogus=*"); - hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0)); - HRN_CFG_LOAD(cfgCmdServerStart, argList); - - // Get pid of this process to identify child process later - pid_t pid = getpid(); - - TEST_RESULT_VOID(cmdServer(2), "server"); - - // If this is a child process then exit immediately - if (pid != getpid()) - exit(0); + HRN_FORK_BEGIN(.timeout = 15000) + { + HRN_FORK_CHILD_BEGIN(.prefix = "server") + { + StringList *argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptTlsServerCaFile, HRN_SERVER_CA); + hrnCfgArgRawZ(argList, cfgOptTlsServerCertFile, HRN_SERVER_CERT); + hrnCfgArgRawZ(argList, cfgOptTlsServerKeyFile, HRN_SERVER_KEY); + hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "bogus=*"); + hrnCfgArgRawFmt(argList, cfgOptTlsServerPort, "%u", hrnServerPort(0)); + HRN_CFG_LOAD(cfgCmdServer, argList); + + // Init exit signal handlers + exitInit(); + + // No log testing needed + harnessLogLevelSet(logLevelError); + + // Get pid of this process to identify child process later + pid_t pid = getpid(); + + TEST_RESULT_VOID(cmdServer(strLstSize(argList), strLstPtr(argList)), "server"); + + // If this is a child process then exit immediately + if (pid != getpid()) + { + HRN_FORK_CHILD_NOTIFY_PUT(); + exit(0); + } + } + HRN_FORK_CHILD_END(); + + HRN_FORK_PARENT_BEGIN(.prefix = "server control") + { + // Wait for forked child processes to exit + HRN_FORK_PARENT_NOTIFY_GET(0); + HRN_FORK_PARENT_NOTIFY_GET(0); + + // Send term to child processes + kill(HRN_FORK_PROCESS_ID(0), SIGTERM); + } + HRN_FORK_PARENT_END(); + } + HRN_FORK_END(); // Wait for child process to exit HRN_FORK_PARENT_NOTIFY_GET(0); diff -Nru pgbackrest-2.36/test/src/module/command/stanzaTest.c pgbackrest-2.37/test/src/module/command/stanzaTest.c --- pgbackrest-2.36/test/src/module/command/stanzaTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/stanzaTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -61,9 +61,7 @@ HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 6569239123849665679})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_96); TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create - one repo, no files exist"); TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'db' on repo1"); @@ -72,11 +70,11 @@ storageHrn, "test.info", "[db]\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.6\"}\n", + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n", .comment = "put archive info to test file"); TEST_RESULT_BOOL( @@ -94,11 +92,11 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n", .comment = "put backup info to test file"); @@ -363,11 +361,11 @@ storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, "[db]\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.6\"}\n", + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n", .comment = "put archive info to file repo1"); TEST_STORAGE_EXISTS( storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE INFO_COPY_EXT, .remove = true, @@ -396,19 +394,19 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n", .comment = "put backup info to file - bad db-id"); TEST_ERROR( cmdStanzaCreate(), FileInvalidError, "backup info file and archive info file do not match\n" - "archive: id = 1, version = 9.6, system-id = 6569239123849665679\n" - "backup : id = 2, version = 9.6, system-id = 6569239123849665679\n" + "archive: id = 1, version = 9.6, system-id = " HRN_PG_SYSTEMID_96_Z "\n" + "backup : id = 2, version = 9.6, system-id = " HRN_PG_SYSTEMID_96_Z "\n" "HINT: this may be a symptom of repository corruption!"); TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'db' on repo1"); @@ -427,22 +425,22 @@ "db-catalog-version=201510051\n" "db-control-version=942\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.5\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.5\"}\n"); HRN_INFO_PUT( storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, "[db]\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.5\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.5\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.5\"}\n"); TEST_ERROR( cmdStanzaCreate(), FileInvalidError, @@ -528,9 +526,7 @@ TEST_TITLE("pgControl and database match"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 6569239123849665699})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_92); harnessPqScriptSet((HarnessPq []) { @@ -568,9 +564,7 @@ TEST_TITLE("pg_control and version mismatch"); // Create pg_control with different version - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_91, .systemId = 6569239123849665699})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_91); harnessPqScriptSet((HarnessPq []) { @@ -588,9 +582,7 @@ TEST_TITLE("pg_control and path mismatch"); // Create pg_control - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 6569239123849665699})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_92); harnessPqScriptSet((HarnessPq []) { @@ -617,14 +609,10 @@ HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); // Create pg_control for primary - HRN_STORAGE_PUT( - storagePgIdxWrite(1), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_92, .systemId = 6569239123849665699})); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_92); // Create pg_control for standby - HRN_STORAGE_PUT( - storagePgIdxWrite(0), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_94, .systemId = 6569239123849665700})); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_94); harnessPqScriptSet((HarnessPq []) { @@ -636,7 +624,7 @@ PgControl pgControl = {0}; TEST_ASSIGN(pgControl, pgValidate(), "validate primary on pg2"); TEST_RESULT_UINT(pgControl.version, PG_VERSION_92, "version set"); - TEST_RESULT_UINT(pgControl.systemId, 6569239123849665699, "systemId set"); + TEST_RESULT_UINT(pgControl.systemId, HRN_PG_SYSTEMID_92, "systemId set"); TEST_RESULT_UINT(pgControl.catalogVersion, 201204301, "catalogVersion set"); } @@ -664,9 +652,7 @@ //-------------------------------------------------------------------------------------------------------------------------- // Create pg_control for the rest of the tests - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 6569239123849665679})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_96); //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("stanza-upgrade - info file mismatch: db-id"); @@ -678,11 +664,11 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n"); // backup info up to date but archive info db-id mismatch @@ -690,17 +676,17 @@ storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, "[db]\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "2={\"db-id\":6569239123849665679,\"db-version\":\"9.6\"}\n"); + "2={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n"); TEST_ERROR( cmdStanzaUpgrade(), FileInvalidError, "backup info file and archive info file do not match\n" - "archive: id = 2, version = 9.6, system-id = 6569239123849665679\n" - "backup : id = 1, version = 9.6, system-id = 6569239123849665679\n" + "archive: id = 2, version = 9.6, system-id = " HRN_PG_SYSTEMID_96_Z "\n" + "backup : id = 1, version = 9.6, system-id = " HRN_PG_SYSTEMID_96_Z "\n" "HINT: this may be a symptom of repository corruption!"); TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'db' on repo1"); @@ -714,23 +700,23 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6569239123849665999," "\"db-version\":\"9.5\"}\n" - "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n"); HRN_INFO_PUT( storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, "[db]\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.5\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.5\"}\n"); + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.5\"}\n"); TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - archive.info file upgraded - version"); TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'db' on repo1"); @@ -739,12 +725,12 @@ storageHrn, "test.info", "[db]\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6569239123849665679,\"db-version\":\"9.5\"}\n" - "2={\"db-id\":6569239123849665679,\"db-version\":\"9.6\"}\n", + "1={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.5\"}\n" + "2={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n", .comment = "put archive info to test file"); TEST_RESULT_BOOL( @@ -766,11 +752,11 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=1\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.5\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.5\"}\n"); TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - backup.info file upgraded - version"); @@ -782,13 +768,13 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" - "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6569239123849665679," + "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.5\"}\n" - "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n", .comment = "put backup info to test file"); @@ -811,13 +797,13 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201510051,\"db-control-version\":942,\"db-system-id\":6569239123849665999," "\"db-version\":\"9.5\"}\n" - "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n"); HRN_INFO_PUT( storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE, @@ -836,12 +822,12 @@ storageHrn, "test.info", "[db]\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-id\":6569239123849665999,\"db-version\":\"9.6\"}\n" - "2={\"db-id\":6569239123849665679,\"db-version\":\"9.6\"}\n", + "2={\"db-id\":" HRN_PG_SYSTEMID_96_Z ",\"db-version\":\"9.6\"}\n", .comment = "put archive info to test file"); TEST_RESULT_BOOL( @@ -879,13 +865,13 @@ "db-catalog-version=201608131\n" "db-control-version=960\n" "db-id=2\n" - "db-system-id=6569239123849665679\n" + "db-system-id=" HRN_PG_SYSTEMID_96_Z "\n" "db-version=\"9.6\"\n" "\n" "[db:history]\n" "1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665999," "\"db-version\":\"9.6\"}\n" - "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665679," + "2={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":" HRN_PG_SYSTEMID_96_Z "," "\"db-version\":\"9.6\"}\n", .comment = "put backup info to test file"); TEST_RESULT_BOOL( @@ -899,7 +885,7 @@ } // ***************************************************************************************************************************** - if (testBegin("cmdStanzaDelete(), stanzaDelete(), manifestDelete()")) + if (testBegin("cmdStanzaDelete(), stanzaDelete()")) { // Load Parameters StringList *argListCmd = strLstNew(); @@ -913,9 +899,7 @@ HRN_CFG_LOAD(cfgCmdStanzaCreate, argList); // Create pg_control for stanza-create - HRN_STORAGE_PUT( - storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 6569239123849665679})); + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_96); TEST_RESULT_VOID(cmdStanzaCreate(), "create a stanza that will not be deleted"); TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'otherstanza' on repo1"); @@ -978,46 +962,6 @@ storageTest, "repo/backup", "otherstanza/\n", .noRecurse=true, .comment = "stanza '" TEST_STANZA "' deleted"); //-------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("stanza-delete - error on file that looks like backup directory"); - - // Create a backup file that matches the regex for a backup directory - HRN_STORAGE_PUT_EMPTY( - storageRepoWrite(), STORAGE_REPO_BACKUP "/20190708-154306F", .comment = "backup file that looks like a directory"); - HRN_STORAGE_PUT_EMPTY(storageHrn, strZ(lockStopFileName(cfgOptionStr(cfgOptStanza))), .comment = "create stop file"); - TEST_ERROR( - cmdStanzaDelete(), FileRemoveError, - "unable to remove '" TEST_PATH "/repo/backup/db/20190708-154306F/backup.manifest': [20] Not a directory"); - HRN_STORAGE_REMOVE(storageTest, "repo/backup/db/20190708-154306F", "cleanup - remove backup file"); - - //-------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("manifestDelete()"); - - // Create backup manifests - HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), STORAGE_REPO_BACKUP "/20190708-154306F/" BACKUP_MANIFEST_FILE); - HRN_STORAGE_PUT_EMPTY( - storageRepoWrite(), STORAGE_REPO_BACKUP "/20190708-154306F_20190716-191726I/" BACKUP_MANIFEST_FILE INFO_COPY_EXT); - HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), STORAGE_REPO_BACKUP "/20190708-154306F_20190716-191800D/" BACKUP_MANIFEST_FILE); - HRN_STORAGE_PUT_EMPTY( - storageRepoWrite(), STORAGE_REPO_BACKUP "/20190708-154306F_20190716-191800D/" BACKUP_MANIFEST_FILE INFO_COPY_EXT); - - TEST_STORAGE_LIST( - storageRepoWrite(), STORAGE_REPO_BACKUP, - "20190708-154306F/\n" - "20190708-154306F/backup.manifest\n" - "20190708-154306F_20190716-191726I/\n" - "20190708-154306F_20190716-191726I/backup.manifest.copy\n" - "20190708-154306F_20190716-191800D/\n" - "20190708-154306F_20190716-191800D/backup.manifest\n" - "20190708-154306F_20190716-191800D/backup.manifest.copy\n"); - - TEST_RESULT_VOID(manifestDelete(storageRepoWrite()), "delete manifests"); - TEST_STORAGE_LIST( - storageRepoWrite(), STORAGE_REPO_BACKUP, - "20190708-154306F/\n" - "20190708-154306F_20190716-191726I/\n" - "20190708-154306F_20190716-191800D/\n", .comment = "all manifest files deleted"); - - //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("stanza-delete - empty directories"); TEST_RESULT_VOID(cmdStanzaDelete(), "remove stanza '" TEST_STANZA "'"); diff -Nru pgbackrest-2.36/test/src/module/command/verifyTest.c pgbackrest-2.37/test/src/module/command/verifyTest.c --- pgbackrest-2.36/test/src/module/command/verifyTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/command/verifyTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -37,14 +37,14 @@ "db-catalog-version=201409291\n" \ "db-control-version=942\n" \ "db-id=1\n" \ - "db-system-id=6625592122879095702\n" \ + "db-system-id=" HRN_PG_SYSTEMID_94_Z "\n" \ "db-version=\"9.4\"\n" #define TEST_BACKUP_DB2_11 \ "db-catalog-version=201707211\n" \ "db-control-version=1100\n" \ "db-id=2\n" \ - "db-system-id=6626363367545678089\n" \ + "db-system-id=" HRN_PG_SYSTEMID_11_Z "\n" \ "db-version=\"11\"\n" #define TEST_BACKUP_DB1_CURRENT_FULL1 \ @@ -77,11 +77,11 @@ "\"option-checksum-page\":true,\"option-compress\":true,\"option-hardlink\":false,\"option-online\":true}\n" #define TEST_BACKUP_DB1_HISTORY \ - "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":6625592122879095702," \ + "1={\"db-catalog-version\":201409291,\"db-control-version\":942,\"db-system-id\":" HRN_PG_SYSTEMID_94_Z "," \ "\"db-version\":\"9.4\"}" #define TEST_BACKUP_DB2_HISTORY \ - "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":6626363367545678089," \ + "2={\"db-catalog-version\":201707211,\"db-control-version\":1100,\"db-system-id\":" HRN_PG_SYSTEMID_11_Z "," \ "\"db-version\":\"11\"}" #define TEST_BACKUP_INFO_MULTI_HISTORY_BASE \ @@ -101,21 +101,21 @@ #define TEST_ARCHIVE_INFO_BASE \ "[db]\n" \ "db-id=1\n" \ - "db-system-id=6625592122879095702\n" \ + "db-system-id=" HRN_PG_SYSTEMID_94_Z "\n" \ "db-version=\"9.4\"\n" \ "\n" \ "[db:history]\n" \ - "1={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}" + "1={\"db-id\":" HRN_PG_SYSTEMID_94_Z ",\"db-version\":\"9.4\"}" #define TEST_ARCHIVE_INFO_MULTI_HISTORY_BASE \ "[db]\n" \ "db-id=2\n" \ - "db-system-id=6626363367545678089\n" \ + "db-system-id=" HRN_PG_SYSTEMID_11_Z "\n" \ "db-version=\"11\"\n" \ "\n" \ "[db:history]\n" \ - "1={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}\n" \ - "2={\"db-id\":6626363367545678089,\"db-version\":\"11\"}" + "1={\"db-id\":" HRN_PG_SYSTEMID_94_Z ",\"db-version\":\"9.4\"}\n" \ + "2={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}" #define TEST_MANIFEST_HEADER \ "[backup]\n" \ @@ -131,7 +131,7 @@ "db-catalog-version=201204301\n" \ "db-control-version=922\n" \ "db-id=1\n" \ - "db-system-id=6625592122879095702\n" \ + "db-system-id=" HRN_PG_SYSTEMID_94_Z "\n" \ "db-version=\"9.2\"\n" #define TEST_MANIFEST_DB_94 \ @@ -140,7 +140,7 @@ "db-catalog-version=201409291\n" \ "db-control-version=942\n" \ "db-id=1\n" \ - "db-system-id=6625592122879095702\n" \ + "db-system-id=" HRN_PG_SYSTEMID_94_Z "\n" \ "db-version=\"9.4\"\n" #define TEST_MANIFEST_OPTION_ALL \ @@ -257,7 +257,7 @@ "P00 WARN: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest.copy'" " for read\n" "P00 ERROR: [028]: '20181119-152138F' may not be recoverable - PG data (id 1, version 9.2, system-id " - "6625592122879095702) is not in the backup.info history, skipping"); + HRN_PG_SYSTEMID_94_Z ") is not in the backup.info history, skipping"); //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("rerun test with db-system-id invalid and no main"); @@ -305,7 +305,7 @@ "db-catalog-version=201409291\n" "db-control-version=942\n" "db-id=0\n" - "db-system-id=6625592122879095702\n" + "db-system-id=" HRN_PG_SYSTEMID_94_Z "\n" "db-version=\"9.4\"\n" TEST_MANIFEST_OPTION_ALL TEST_MANIFEST_TARGET @@ -326,7 +326,7 @@ "P00 WARN: unable to open missing file '" TEST_PATH "/repo/backup/db/20181119-152138F/backup.manifest' for read\n" "P00 WARN: 20181119-152138F/backup.manifest is missing or unusable, using copy\n" "P00 ERROR: [028]: '20181119-152138F' may not be recoverable - PG data (id 0, version 9.4, system-id " - "6625592122879095702) is not in the backup.info history, skipping"); + HRN_PG_SYSTEMID_94_Z ") is not in the backup.info history, skipping"); //-------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("missing main manifest, errored copy"); @@ -567,11 +567,11 @@ archiveInfo, infoArchiveNewLoad(ioBufferReadNew(harnessInfoChecksumZ( "[db]\n" "db-id=2\n" - "db-system-id=6626363367545678089\n" + "db-system-id=" HRN_PG_SYSTEMID_11_Z "\n" "db-version=\"11\"\n" "\n" "[db:history]\n" - "2={\"db-id\":6626363367545678089,\"db-version\":\"11\"}"))), "archive.info missing history"); + "2={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}"))), "archive.info missing history"); TEST_ERROR( verifyPgHistory(infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FormatError, @@ -584,12 +584,12 @@ archiveInfo, infoArchiveNewLoad(ioBufferReadNew(harnessInfoChecksumZ( "[db]\n" "db-id=2\n" - "db-system-id=6626363367545678089\n" + "db-system-id=" HRN_PG_SYSTEMID_11_Z "\n" "db-version=\"11\"\n" "\n" "[db:history]\n" "1={\"db-id\":6625592122879095777,\"db-version\":\"9.4\"}\n" - "2={\"db-id\":6626363367545678089,\"db-version\":\"11\"}"))), "archive.info history system id mismatch"); + "2={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}"))), "archive.info history system id mismatch"); TEST_ERROR( verifyPgHistory(infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FormatError, @@ -602,12 +602,12 @@ archiveInfo, infoArchiveNewLoad(ioBufferReadNew(harnessInfoChecksumZ( "[db]\n" "db-id=2\n" - "db-system-id=6626363367545678089\n" + "db-system-id=" HRN_PG_SYSTEMID_11_Z "\n" "db-version=\"11\"\n" "\n" "[db:history]\n" - "1={\"db-id\":6625592122879095702,\"db-version\":\"9.5\"}\n" - "2={\"db-id\":6626363367545678089,\"db-version\":\"11\"}"))), "archive.info history version mismatch"); + "1={\"db-id\":" HRN_PG_SYSTEMID_94_Z ",\"db-version\":\"9.5\"}\n" + "2={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}"))), "archive.info history version mismatch"); TEST_ERROR( verifyPgHistory(infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FormatError, @@ -620,12 +620,12 @@ archiveInfo, infoArchiveNewLoad(ioBufferReadNew(harnessInfoChecksumZ( "[db]\n" "db-id=2\n" - "db-system-id=6626363367545678089\n" + "db-system-id=" HRN_PG_SYSTEMID_11_Z "\n" "db-version=\"11\"\n" "\n" "[db:history]\n" - "3={\"db-id\":6625592122879095702,\"db-version\":\"9.4\"}\n" - "2={\"db-id\":6626363367545678089,\"db-version\":\"11\"}"))), "archive.info history id mismatch"); + "3={\"db-id\":" HRN_PG_SYSTEMID_94_Z ",\"db-version\":\"9.4\"}\n" + "2={\"db-id\":" HRN_PG_SYSTEMID_11_Z ",\"db-version\":\"11\"}"))), "archive.info history id mismatch"); TEST_ERROR( verifyPgHistory(infoArchivePg(archiveInfo), infoBackupPg(backupInfo)), FormatError, @@ -804,8 +804,8 @@ "P00 WARN: invalid checksum, actual 'e056f784a995841fd4e2802b809299b8db6803a2' but expected 'BOGUS'" " /archive.info\n" "P00 ERROR: [029]: backup info file and archive info file do not match\n" - " archive: id = 1, version = 9.4, system-id = 6625592122879095702\n" - " backup : id = 2, version = 11, system-id = 6626363367545678089\n" + " archive: id = 1, version = 9.4, system-id = " HRN_PG_SYSTEMID_94_Z "\n" + " backup : id = 2, version = 11, system-id = " HRN_PG_SYSTEMID_11_Z "\n" " HINT: this may be a symptom of repository corruption!"); //-------------------------------------------------------------------------------------------------------------------------- @@ -926,8 +926,7 @@ Buffer *walBuffer = bufNew((size_t)(1024 * 1024)); bufUsedSet(walBuffer, bufSize(walBuffer)); memset(bufPtr(walBuffer), 0, bufSize(walBuffer)); - hrnPgWalToBuffer( - (PgWal){.version = PG_VERSION_11, .systemId = 6626363367545678089, .size = 1024 * 1024}, walBuffer); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer); const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer))); HRN_STORAGE_PUT( @@ -1419,8 +1418,7 @@ Buffer *walBuffer = bufNew((size_t)(1024 * 1024)); bufUsedSet(walBuffer, bufSize(walBuffer)); memset(bufPtr(walBuffer), 0, bufSize(walBuffer)); - hrnPgWalToBuffer( - (PgWal){.version = PG_VERSION_11, .systemId = 6626363367545678089, .size = 1024 * 1024}, walBuffer); + hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer); const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer))); HRN_STORAGE_PUT( diff -Nru pgbackrest-2.36/test/src/module/common/compressTest.c pgbackrest-2.37/test/src/module/common/compressTest.c --- pgbackrest-2.36/test/src/module/common/compressTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/compressTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -335,9 +335,9 @@ { TEST_TITLE("compressTypeEnum()"); - TEST_RESULT_UINT(compressTypeEnum(STRDEF("none")), compressTypeNone, "none enum"); - TEST_RESULT_UINT(compressTypeEnum(STRDEF("gz")), compressTypeGz, "gz enum"); - TEST_ERROR(compressTypeEnum(STRDEF(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'"); + TEST_RESULT_UINT(compressTypeEnum(strIdFromZ("none")), compressTypeNone, "none enum"); + TEST_RESULT_UINT(compressTypeEnum(strIdFromZ("gz")), compressTypeGz, "gz enum"); + TEST_ERROR(compressTypeEnum(strIdFromZ(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("compressTypeStr()"); diff -Nru pgbackrest-2.36/test/src/module/common/errorTest.c pgbackrest-2.37/test/src/module/common/errorTest.c --- pgbackrest-2.36/test/src/module/common/errorTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/errorTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -96,7 +96,7 @@ } FINALLY() { - assert(errorContext.tryList[1].state == errorStateFinal); + assert(errorContext.tryList[1].state == errorStateTry); finallyDone = true; } TRY_END(); @@ -195,7 +195,7 @@ { assert(testErrorHandlerTryDepth == 1); assert(errorTryDepth() == 1); - assert(errorContext.tryList[1].state == errorStateCatch); + assert(errorContext.tryList[1].state == errorStateEnd); assert(strlen(errorMessage()) == sizeof(messageBuffer) - 1); catchDone = true; diff -Nru pgbackrest-2.36/test/src/module/common/ioHttpTest.c pgbackrest-2.37/test/src/module/common/ioHttpTest.c --- pgbackrest-2.36/test/src/module/common/ioHttpTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/ioHttpTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -581,7 +581,7 @@ httpRequestError(request, response), ProtocolError, "HTTP request failed with 404 (Not Found):\n" "*** Path/Query ***:\n" - "/"); + "GET /"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("error with content"); @@ -612,7 +612,7 @@ httpRequestError(request, response), ProtocolError, "HTTP request failed with 403:\n" "*** Path/Query ***:\n" - "/?a=b\n" + "GET /?a=b\n" "*** Request Headers ***:\n" "hdr1: 1\n" "hdr2: \n" diff -Nru pgbackrest-2.36/test/src/module/common/ioTest.c pgbackrest-2.37/test/src/module/common/ioTest.c --- pgbackrest-2.36/test/src/module/common/ioTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/ioTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -677,8 +677,8 @@ THROW_ON_SYS_ERROR((flags = fcntl(fd, F_GETFL)) == -1, ProtocolError, "unable to get flags"); THROW_ON_SYS_ERROR(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1, ProtocolError, "unable to set O_NONBLOCK"); - // Attempt connection - CHECK(connect(fd, hostBadAddress->ai_addr, hostBadAddress->ai_addrlen) == -1); + // Make sure the bad address does not work before using it for testing + ASSERT(connect(fd, hostBadAddress->ai_addr, hostBadAddress->ai_addrlen) == -1); // Create file descriptor write and wait for timeout IoWrite *write = NULL; diff -Nru pgbackrest-2.36/test/src/module/common/ioTlsTest.c pgbackrest-2.37/test/src/module/common/ioTlsTest.c --- pgbackrest-2.36/test/src/module/common/ioTlsTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/ioTlsTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -128,6 +128,15 @@ "-----END CERTIFICATE-----"; /*********************************************************************************************************************************** +Test signal handler that does nothing +***********************************************************************************************************************************/ +static void +testSignalHandler(const int signalType) +{ + (void)signalType; +} + +/*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ static void @@ -255,8 +264,8 @@ // --------------------------------------------------------------------------------------------------------------------- TEST_TITLE("connect to non-blocking socket to test write ready"); - // Attempt connection - CHECK(connect(fd, hostBadAddress->ai_addr, hostBadAddress->ai_addrlen) == -1); + // Make sure the bad address does not work before using it for testing + ASSERT(connect(fd, hostBadAddress->ai_addr, hostBadAddress->ai_addrlen) == -1); // Create socket session and wait for timeout IoSession *session = NULL; @@ -299,7 +308,7 @@ } // ***************************************************************************************************************************** - if (testBegin("SocketClient")) + if (testBegin("SocketClient/SocketServer")) { IoClient *client = NULL; @@ -311,6 +320,37 @@ // This address should not be in use in a test environment -- if it is the test will fail TEST_ASSIGN(client, sckClientNew(STRDEF("172.31.255.255"), hrnServerPort(0), 100, 100), "new client"); TEST_ERROR_FMT(ioClientOpen(client), HostConnectError, "timeout connecting to '172.31.255.255:%u'", hrnServerPort(0)); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("sckServerAccept() returns NULL on interrupt"); + + HRN_FORK_BEGIN(.timeout = 5000) + { + HRN_FORK_CHILD_BEGIN(.prefix = "sighup server") + { + // Ignore SIGHUP + sigaction(SIGHUP, &(struct sigaction){.sa_handler = testSignalHandler}, NULL); + + // Wait for connection. Use port 1 to avoid port conflicts later. + IoServer *server = sckServerNew(STRDEF("127.0.0.1"), hrnServerPort(1), 5000); + HRN_FORK_CHILD_NOTIFY_PUT(); + + TEST_RESULT_PTR(ioServerAccept(server, NULL), NULL, "connection interrupted"); + } + HRN_FORK_CHILD_END(); + + HRN_FORK_PARENT_BEGIN(.prefix = "sighup client") + { + // Wait for client to be ready but also sleep a bit more to allow accept to initialize + HRN_FORK_PARENT_NOTIFY_GET(0); + sleep(1); + + // Send SIGHUP and the client should exit + kill(HRN_FORK_PROCESS_ID(0), SIGHUP); + } + HRN_FORK_PARENT_END(); + } + HRN_FORK_END(); } // Additional coverage not provided by testing with actual certificates @@ -338,17 +378,13 @@ // Connection errors // ------------------------------------------------------------------------------------------------------------------------- TEST_ASSIGN( - client, - tlsClientNew(sckClientNew(STRDEF("99.99.99.99.99"), 7777, 0, 0), STRDEF("X"), 0, 0, true, NULL, NULL, NULL, NULL), - "new client"); + client, tlsClientNewP(sckClientNew(STRDEF("99.99.99.99.99"), 7777, 0, 0), STRDEF("X"), 0, 0, true), "new client"); TEST_RESULT_STR_Z(ioClientName(client), "99.99.99.99.99:7777", " check name"); TEST_ERROR( ioClientOpen(client), HostConnectError, "unable to get address for '99.99.99.99.99': [-2] Name or service not known"); TEST_ASSIGN( - client, - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 100, 100), STRDEF("X"), 100, 100, true, NULL, NULL, NULL, NULL), + client, tlsClientNewP(sckClientNew(STRDEF("localhost"), hrnServerPort(0), 100, 100), STRDEF("X"), 100, 100, true), "new client"); TEST_ERROR_FMT( ioClientOpen(client), HostConnectError, "unable to connect to 'localhost:%u': [111] Connection refused", @@ -359,9 +395,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, STRDEF("bogus.crt"), - STRDEF("/bogus"), NULL, NULL)), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .caFile = STRDEF("bogus.crt"), .caPath = STRDEF("/bogus"))), CryptoError, "unable to set user-defined CA certificate location: [33558530] No such file or directory"); // ------------------------------------------------------------------------------------------------------------------------- @@ -369,9 +405,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF("/bogus"), STRDEF("/bogus"))), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF("/bogus"), .keyFile = STRDEF("/bogus"))), CryptoError, "unable to load cert file '/bogus': [33558530] No such file or directory"); // ------------------------------------------------------------------------------------------------------------------------- @@ -379,9 +415,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF("/bogus"))), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF("/bogus"))), CryptoError, "unable to load key file '/bogus': [33558530] No such file or directory"); // ------------------------------------------------------------------------------------------------------------------------- @@ -389,9 +425,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(HRN_SERVER_KEY))), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(HRN_SERVER_KEY))), CryptoError, "unable to load key file '" HRN_PATH_REPO "/test/certificate/pgbackrest-test-server.key': [185073780] key values" " mismatch"); @@ -404,17 +440,17 @@ TRY_BEGIN() { TEST_ERROR( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(TEST_PATH "/client-pwd.key")), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(TEST_PATH "/client-pwd.key")), CryptoError, "unable to load key file '" TEST_PATH "/client-pwd.key': [101077092] bad decrypt"); } CATCH(TestError) { TEST_ERROR( // {uncovered - 32-bit error} - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(TEST_PATH "/client-pwd.key")), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(TEST_PATH "/client-pwd.key")), CryptoError, "unable to load key file '" TEST_PATH "/client-pwd.key': [151429224] bad password read"); } TRY_END(); @@ -428,9 +464,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(TEST_PATH "/client-bad-perm.key"))), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(TEST_PATH "/client-bad-perm.key"))), FileReadError, "key file '" TEST_PATH "/client-bad-perm.key' has group or other permissions\n" "HINT: file must have permissions u=rw (0600) or less if owned by the '" TEST_USER "' user\n" @@ -447,9 +483,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(TEST_PATH "/client-bad-perm.key"))), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(TEST_PATH "/client-bad-perm.key"))), FileReadError, "key file '" TEST_PATH "/client-bad-perm.key' must be owned by the '" TEST_USER "' user or root"); // ------------------------------------------------------------------------------------------------------------------------- @@ -459,9 +495,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, NULL, - STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(TEST_PATH "/client-bad-perm.key"))), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(TEST_PATH "/client-bad-perm.key"))), FileReadError, "key file '" TEST_PATH "/client-bad-perm.key' has group or other permissions\n" "HINT: file must have permissions u=rw (0600) or less if owned by the '" TEST_USER "' user\n" @@ -502,9 +538,9 @@ TEST_ERROR_FMT( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, NULL, - STRDEF("/bogus"), NULL, NULL)), + tlsClientNewP( + sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, + .caPath = STRDEF("/bogus"))), CryptoError, "unable to verify certificate presented by 'localhost:%u': [20] unable to get local issuer certificate", hrnServerPort(0)); @@ -517,9 +553,9 @@ TEST_RESULT_VOID( ioClientOpen( - tlsClientNew( + tlsClientNewP( sckClientNew(STRDEF("test.pgbackrest.org"), hrnServerPort(0), 5000, 5000), - STRDEF("test.pgbackrest.org"), 0, 0, true, STRDEF(HRN_SERVER_CA), NULL, NULL, NULL)), + STRDEF("test.pgbackrest.org"), 0, 0, true, .caFile = STRDEF(HRN_SERVER_CA))), "open connection"); // ----------------------------------------------------------------------------------------------------------------- @@ -530,9 +566,9 @@ TEST_RESULT_VOID( ioClientOpen( - tlsClientNew( + tlsClientNewP( sckClientNew(STRDEF("host.test2.pgbackrest.org"), hrnServerPort(0), 5000, 5000), - STRDEF("host.test2.pgbackrest.org"), 0, 0, true, STRDEF(HRN_SERVER_CA), NULL, NULL, NULL)), + STRDEF("host.test2.pgbackrest.org"), 0, 0, true, .caFile = STRDEF(HRN_SERVER_CA))), "open connection"); // ----------------------------------------------------------------------------------------------------------------- @@ -543,9 +579,9 @@ TEST_ERROR( ioClientOpen( - tlsClientNew( + tlsClientNewP( sckClientNew(STRDEF("test3.pgbackrest.org"), hrnServerPort(0), 5000, 5000), - STRDEF("test3.pgbackrest.org"), 0, 0, true, STRDEF(HRN_SERVER_CA), NULL, NULL, NULL)), + STRDEF("test3.pgbackrest.org"), 0, 0, true, .caFile = STRDEF(HRN_SERVER_CA))), CryptoError, "unable to find hostname 'test3.pgbackrest.org' in certificate common name or subject alternative names"); @@ -557,9 +593,9 @@ TEST_ERROR_FMT( ioClientOpen( - tlsClientNew( + tlsClientNewP( sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, true, - STRDEF(HRN_SERVER_CERT), NULL, NULL, NULL)), + .caFile = STRDEF(HRN_SERVER_CERT))), CryptoError, "unable to verify certificate presented by 'localhost:%u': [20] unable to get local issuer certificate", hrnServerPort(0)); @@ -572,10 +608,8 @@ TEST_RESULT_VOID( ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, false, NULL, NULL, - NULL, NULL)), - "open connection"); + tlsClientNewP(sckClientNew(STRDEF("localhost"), hrnServerPort(0), 5000, 5000), STRDEF("X"), 0, 0, false)), + "open connection"); // ----------------------------------------------------------------------------------------------------------------- hrnServerScriptEnd(tls); @@ -662,9 +696,10 @@ TEST_ASSIGN( clientSession, ioClientOpen( - tlsClientNew( + tlsClientNewP( sckClientNew(STRDEF("127.0.0.1"), hrnServerPort(0), 5000, 5000), STRDEF("127.0.0.1"), 5000, 5000, - true, NULL, NULL, STRDEF(TEST_PATH "/client-bad-ca.crt"), STRDEF(HRN_SERVER_CLIENT_KEY))), + true, .certFile = STRDEF(TEST_PATH "/client-bad-ca.crt"), + .keyFile = STRDEF(HRN_SERVER_CLIENT_KEY))), "client open"); TEST_ERROR( @@ -680,9 +715,9 @@ TEST_ASSIGN( clientSession, ioClientOpen( - tlsClientNew( + tlsClientNewP( sckClientNew(STRDEF("127.0.0.1"), hrnServerPort(0), 5000, 5000), STRDEF("127.0.0.1"), 5000, 5000, true, - NULL, NULL, STRDEF(HRN_SERVER_CLIENT_CERT), STRDEF(HRN_SERVER_CLIENT_KEY))), + .certFile = STRDEF(HRN_SERVER_CLIENT_CERT), .keyFile = STRDEF(HRN_SERVER_CLIENT_KEY))), "client open"); Buffer *buffer = bufNew(7); @@ -697,9 +732,9 @@ TEST_ASSIGN( clientSession, ioClientOpen( - tlsClientNew( - sckClientNew(STRDEF("127.0.0.1"), hrnServerPort(0), 5000, 5000), STRDEF("127.0.0.1"), 5000, 5000, true, - NULL, NULL, NULL, NULL)), + tlsClientNewP( + sckClientNew(STRDEF("127.0.0.1"), hrnServerPort(0), 5000, 5000), STRDEF("127.0.0.1"), 5000, 5000, + true)), "client open"); buffer = bufNew(8); @@ -738,9 +773,8 @@ TEST_ASSIGN( client, - tlsClientNew( - sckClientNew(hrnServerHost(), hrnServerPort(0), 5000, 5000), hrnServerHost(), 0, 0, TEST_IN_CONTAINER, NULL, - NULL, NULL, NULL), + tlsClientNewP( + sckClientNew(hrnServerHost(), hrnServerPort(0), 5000, 5000), hrnServerHost(), 0, 0, TEST_IN_CONTAINER), "new client"); hrnServerScriptAccept(tls); diff -Nru pgbackrest-2.36/test/src/module/common/logTest.c pgbackrest-2.37/test/src/module/common/logTest.c --- pgbackrest-2.36/test/src/module/common/logTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/logTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -102,10 +102,10 @@ // ***************************************************************************************************************************** if (testBegin("logLevelEnum() and logLevelStr()")) { - TEST_ERROR(logLevelEnum(BOGUS_STR), AssertError, "log level 'BOGUS' not found"); - TEST_RESULT_INT(logLevelEnum("OFF"), logLevelOff, "log level 'OFF' found"); - TEST_RESULT_INT(logLevelEnum("info"), logLevelInfo, "log level 'info' found"); - TEST_RESULT_INT(logLevelEnum("TRACE"), logLevelTrace, "log level 'TRACE' found"); + TEST_ERROR(logLevelEnum(strIdFromZ(BOGUS_STR)), AssertError, "invalid log level"); + TEST_RESULT_INT(logLevelEnum(strIdFromZ("off")), logLevelOff, "log level 'OFF' found"); + TEST_RESULT_INT(logLevelEnum(strIdFromZ("info")), logLevelInfo, "log level 'info' found"); + TEST_RESULT_INT(logLevelEnum(strIdFromZ("trace")), logLevelTrace, "log level 'TRACE' found"); TEST_ERROR(logLevelStr(999), AssertError, "assertion 'logLevel <= LOG_LEVEL_MAX' failed"); TEST_RESULT_Z(logLevelStr(logLevelOff), "OFF", "log level 'OFF' found"); diff -Nru pgbackrest-2.36/test/src/module/common/typeListTest.c pgbackrest-2.37/test/src/module/common/typeListTest.c --- pgbackrest-2.36/test/src/module/common/typeListTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/typeListTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -203,7 +203,7 @@ for (int listIdx = 0; listIdx < testMax; listIdx++) lstAdd(list, &listIdx); - CHECK(lstSize(list) == (unsigned int)testMax); + ASSERT(lstSize(list) == (unsigned int)testMax); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("search ascending sort"); @@ -211,7 +211,7 @@ lstSort(list, sortOrderAsc); for (int listIdx = 0; listIdx < testMax; listIdx++) - CHECK(*(int *)lstFind(list, &listIdx) == listIdx); + ASSERT(*(int *)lstFind(list, &listIdx) == listIdx); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("search descending sort"); @@ -219,7 +219,7 @@ lstSort(list, sortOrderDesc); for (int listIdx = 0; listIdx < testMax; listIdx++) - CHECK(*(int *)lstFind(list, &listIdx) == listIdx); + ASSERT(*(int *)lstFind(list, &listIdx) == listIdx); } FUNCTION_HARNESS_RETURN_VOID(); diff -Nru pgbackrest-2.36/test/src/module/common/typeStringTest.c pgbackrest-2.37/test/src/module/common/typeStringTest.c --- pgbackrest-2.36/test/src/module/common/typeStringTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/common/typeStringTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -553,13 +553,13 @@ #define TEST_STR5ID12 (TEST_STR5ID11 | (uint64_t)('6' - 24) << 59) #define TEST_STR5ID13 (TEST_STR5ID12 | STRING_ID_PREFIX) - TEST_RESULT_UINT(strIdFromZN(stringIdBit5, "a", 1), TEST_STR5ID1, "5 bits 1 char"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit5, "abc-zk", 6), TEST_STR5ID6, "5 bits 6 chars"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit5, "abc-zkz2-y56", 12), TEST_STR5ID12, "5 bits 12 chars"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit5, "abc-zkz2-y56?", 13), TEST_STR5ID13, "5 bits 13 chars"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit5, "abc-zkz2-y56??", 14), TEST_STR5ID13, "5 bits 14 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit5, "a", 1), TEST_STR5ID1, "5 bits 1 char"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit5, "abc-zk", 6), TEST_STR5ID6, "5 bits 6 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit5, "abc-zkz2-y56", 12), TEST_STR5ID12, "5 bits 12 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit5, "abc-zkz2-y56?", 13), TEST_STR5ID13, "5 bits 13 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit5, "abc-zkz2-y56??", 14), TEST_STR5ID13, "5 bits 14 chars"); - TEST_ERROR(strIdFromZN(stringIdBit5, "AB", 2), FormatError, "'A' is invalid for 5-bit encoding in 'AB'"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit5, "AB", 2), 0, "'A' is invalid for 5-bit encoding in 'AB'"); #define TEST_STR6ID1 (stringIdBit6 | (uint16_t)('a' - 96) << 4) #define TEST_STR6ID2 (TEST_STR6ID1 | (uint16_t)('b' - 96) << 10) @@ -573,13 +573,13 @@ #define TEST_STR6ID10 (TEST_STR6ID9 | (uint64_t)('9' - 20) << 58) #define TEST_STR6ID11 (TEST_STR6ID10 | STRING_ID_PREFIX) - TEST_RESULT_UINT(strIdFromZN(stringIdBit6, "a", 1), TEST_STR6ID1, "6 bits 1 char"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit6, "abC-4", 5), TEST_STR6ID5, "6 bits 5 chars"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit6, "abC-40MzZ9", 10), TEST_STR6ID10, "6 bits 10 chars"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit6, "abC-40MzZ9?", 11), TEST_STR6ID11, "6 bits 11 chars"); - TEST_RESULT_UINT(strIdFromZN(stringIdBit6, "abC-40MzZ9??", 12), TEST_STR6ID11, "6 bits 12 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit6, "a", 1), TEST_STR6ID1, "6 bits 1 char"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit6, "abC-4", 5), TEST_STR6ID5, "6 bits 5 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit6, "abC-40MzZ9", 10), TEST_STR6ID10, "6 bits 10 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit6, "abC-40MzZ9?", 11), TEST_STR6ID11, "6 bits 11 chars"); + TEST_RESULT_UINT(strIdBitFromZN(stringIdBit6, "abC-40MzZ9??", 12), TEST_STR6ID11, "6 bits 12 chars"); - TEST_ERROR(strIdFromZN(stringIdBit6, "|B", 2), FormatError, "'|' is invalid for 6-bit encoding in '|B'"); + TEST_RESULT_UINT(strIdFromZN("|B", 2, false), 0, "'|' is invalid for 6-bit encoding in '|B'"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("STRID*()"); @@ -590,12 +590,17 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("strIdFromStr()"); - TEST_RESULT_UINT(strIdFromStr(stringIdBit5, STRDEF("abc-")), TEST_STR5ID4, "5 bits 4 chars"); + TEST_RESULT_UINT(strIdFromStr(STRDEF("abc-")), TEST_STR5ID4, "5 bits 4 chars"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("strIdFromZ()"); - TEST_RESULT_UINT(strIdFromZ(stringIdBit6, "abC-"), TEST_STR6ID4, "6 bits 4 chars"); + TEST_RESULT_UINT(strIdFromZ("abC-"), TEST_STR6ID4, "6 bits 4 chars"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on invalid encoding"); + + TEST_ERROR(strIdFromZ("abc?"), FormatError, "'abc?' contains invalid characters"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("strIdToZN()"); diff -Nru pgbackrest-2.36/test/src/module/config/execTest.c pgbackrest-2.37/test/src/module/config/execTest.c --- pgbackrest-2.36/test/src/module/config/execTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/config/execTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -18,6 +18,7 @@ StringList *argList = strLstNew(); hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, "5"); + hrnCfgArgRawZ(argList, cfgOptBufferSize, "64KiB"); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo"); hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, TEST_PATH "/db path"); hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 2, "/db2"); @@ -34,14 +35,14 @@ TEST_RESULT_STRLST_Z( cfgExecParam(cfgCmdArchiveGet, cfgCmdRoleAsync, NULL, false, true), - "--archive-async\n--no-config\n--exec-id=1-test\n--log-subprocess\n--reset-neutral-umask\n" + "--archive-async\n--buffer-size=64KiB\n--no-config\n--exec-id=1-test\n--log-subprocess\n--reset-neutral-umask\n" "--pg1-path=\"" TEST_PATH "/db path\"\n--pg2-path=/db2\n--repo1-path=" TEST_PATH "/repo\n--stanza=test1\n" "archive-get:async\n", "exec archive-get -> archive-get:async"); TEST_RESULT_STRLST_Z( cfgExecParam(cfgCmdBackup, cfgCmdRoleMain, NULL, false, false), - "--archive-timeout=5\n--no-config\n--exec-id=1-test\n--log-subprocess\n--reset-neutral-umask\n" + "--archive-timeout=5\n--buffer-size=64KiB\n--no-config\n--exec-id=1-test\n--log-subprocess\n--reset-neutral-umask\n" "--pg1-path=" TEST_PATH "/db path\n--pg2-path=/db2\n--repo1-path=" TEST_PATH "/repo\n--stanza=test1\nbackup\n", "exec archive-get -> backup"); diff -Nru pgbackrest-2.36/test/src/module/config/loadTest.c pgbackrest-2.37/test/src/module/config/loadTest.c --- pgbackrest-2.36/test/src/module/config/loadTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/config/loadTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -499,7 +499,7 @@ hrnCfgArgRawNegate(argList, cfgOptCompress); HRN_CFG_LOAD(cfgCmdArchivePush, argList); - TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCompressType), "none", "compress-type=none"); + TEST_RESULT_UINT(cfgOptionStrId(cfgOptCompressType), CFGOPTVAL_COMPRESS_TYPE_NONE, "compress-type=none"); TEST_RESULT_INT(cfgOptionInt(cfgOptCompressLevel), 0, "compress-level=0"); TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, "compress is not valid"); @@ -512,7 +512,7 @@ hrnCfgArgRawZ(argList, cfgOptCompressLevel, "9"); HRN_CFG_LOAD(cfgCmdArchivePush, argList); - TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCompressType), "gz", "compress-type=gz"); + TEST_RESULT_UINT(cfgOptionStrId(cfgOptCompressType), CFGOPTVAL_COMPRESS_TYPE_GZ, "compress-type=gz"); TEST_RESULT_INT(cfgOptionInt(cfgOptCompressLevel), 9, "compress-level=9"); TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, "compress is not valid"); @@ -525,7 +525,7 @@ hrnCfgArgRawZ(argList, cfgOptCompressType, "gz"); HRN_CFG_LOAD(cfgCmdArchivePush, argList); - TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCompressType), "gz", "compress-type=gz"); + TEST_RESULT_UINT(cfgOptionStrId(cfgOptCompressType), CFGOPTVAL_COMPRESS_TYPE_GZ, "compress-type=gz"); TEST_RESULT_INT(cfgOptionInt(cfgOptCompressLevel), 6, "compress-level=6"); TEST_RESULT_BOOL(cfgOptionValid(cfgOptCompress), false, "compress is not valid"); @@ -550,7 +550,7 @@ // Only the error case is tested here, success is tested in cfgLoad() TEST_RESULT_VOID(cfgLoadLogFile(), "attempt to open bogus log file"); - TEST_RESULT_STR_Z(cfgOptionStr(cfgOptLogLevelFile), "off", "log-level-file should now be off"); + TEST_RESULT_UINT(cfgOptionStrId(cfgOptLogLevelFile), CFGOPTVAL_LOG_LEVEL_FILE_OFF, "log-level-file should now be off"); } // ***************************************************************************************************************************** diff -Nru pgbackrest-2.36/test/src/module/config/parseTest.c pgbackrest-2.37/test/src/module/config/parseTest.c --- pgbackrest-2.36/test/src/module/config/parseTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/config/parseTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -593,14 +593,15 @@ // ***************************************************************************************************************************** if (testBegin("convertToByte()")) { - TEST_ERROR(sizeQualifierToMultiplier('w'), AssertError, "'w' is not a valid size qualifier"); - TEST_RESULT_UINT(convertToByte(STRDEF("10B")), 10, "10B"); - TEST_RESULT_UINT(convertToByte(STRDEF("1k")), 1024, "1k"); - TEST_RESULT_UINT(convertToByte(STRDEF("5G")), (uint64_t)5 * 1024 * 1024 * 1024, "5G"); - TEST_RESULT_UINT(convertToByte(STRDEF("3Tb")), (uint64_t)3 * 1024 * 1024 * 1024 * 1024, "3Tb"); - TEST_RESULT_UINT(convertToByte(STRDEF("11")), 11, "11 - no qualifier, default bytes"); - TEST_RESULT_UINT(convertToByte(STRDEF("4pB")), 4503599627370496, "4pB"); - TEST_RESULT_UINT(convertToByte(STRDEF("15MB")), (uint64_t)15 * 1024 * 1024, "15MB"); + TEST_ERROR(cfgParseSizeQualifier('w'), AssertError, "'w' is not a valid size qualifier"); + TEST_RESULT_INT(cfgParseSize(STRDEF("10B")), 10, "10B"); + TEST_RESULT_INT(cfgParseSize(STRDEF("1k")), 1024, "1k"); + TEST_RESULT_INT(cfgParseSize(STRDEF("1KiB")), 1024, "1KiB"); + TEST_RESULT_INT(cfgParseSize(STRDEF("5G")), (uint64_t)5 * 1024 * 1024 * 1024, "5G"); + TEST_RESULT_INT(cfgParseSize(STRDEF("3Tb")), (uint64_t)3 * 1024 * 1024 * 1024 * 1024, "3Tb"); + TEST_RESULT_INT(cfgParseSize(STRDEF("11")), 11, "11 - no qualifier, default bytes"); + TEST_RESULT_INT(cfgParseSize(STRDEF("4pB")), 4503599627370496, "4pB"); + TEST_RESULT_INT(cfgParseSize(STRDEF("15MB")), (uint64_t)15 * 1024 * 1024, "15MB"); } // ***************************************************************************************************************************** @@ -627,14 +628,69 @@ "option '-bogus' must begin with --"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("error when option argument not allowed"); + TEST_TITLE("error when option argument is invalid"); argList = strLstNew(); strLstAddZ(argList, TEST_BACKREST_EXE); hrnCfgArgRawZ(argList, cfgOptOnline, "bogus"); TEST_ERROR( + configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionInvalidValueError, + "boolean option '--online' argument must be 'y' or 'n'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("boolean option with negation argument"); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + hrnCfgArgRawZ(argList, cfgOptNeutralUmask, "n"); + strLstAddZ(argList, CFGCMD_BACKUP); + TEST_ERROR( + configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionRequiredError, + "backup command requires option: stanza"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("boolean option with affirmative argument"); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + hrnCfgArgRawZ(argList, cfgOptNeutralUmask, "y"); + strLstAddZ(argList, CFGCMD_BACKUP); + TEST_ERROR( + configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionRequiredError, + "backup command requires option: stanza"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error when negated boolean option with negative argument"); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + strLstAddZ(argList, "--no-neutral-umask=n"); + strLstAddZ(argList, CFGCMD_BACKUP); + TEST_ERROR( + configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionInvalidError, + "option 'no-neutral-umask' does not allow an argument"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error when negated boolean option with affirmative argument"); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + strLstAddZ(argList, "--no-online=y"); + strLstAddZ(argList, CFGCMD_BACKUP); + TEST_ERROR( configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionInvalidError, - "option 'online' does not allow an argument"); + "option 'no-online' does not allow an argument"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error when argumentless option is passed with an argument"); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + strLstAddZ(argList, "--reset-pg1-host=xxx"); + strLstAddZ(argList, CFGCMD_BACKUP); + TEST_ERROR( + configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionInvalidError, + "option 'reset-pg1-host' does not allow an argument"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("invalid command"); @@ -958,10 +1014,10 @@ strLstAddZ(argList, TEST_BACKREST_EXE); strLstAddZ(argList, TEST_COMMAND_BACKUP); hrnCfgArgRawZ(argList, cfgOptStanza, "db"); - hrnCfgArgRawZ(argList, cfgOptManifestSaveThreshold, "9999999999999999999p"); + hrnCfgArgRawZ(argList, cfgOptManifestSaveThreshold, "999999999999999999p"); TEST_ERROR( configParse(storageTest, strLstSize(argList), strLstPtr(argList), false), OptionInvalidValueError, - "'9999999999999999999p' is out of range for 'manifest-save-threshold' option"); + "'999999999999999999p' is out of range for 'manifest-save-threshold' option"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("value missing"); @@ -1479,11 +1535,10 @@ TEST_RESULT_BOOL(cfgOptionNegate(cfgOptConfig), true, "config is negated"); TEST_RESULT_INT(cfgOptionSource(cfgOptStanza), cfgSourceParam, "stanza is source param"); TEST_RESULT_STR_Z(cfgOptionStr(cfgOptStanza), "db", "stanza is set"); - TEST_RESULT_UINT(cfgOptionStrId(cfgOptStanza), strIdFromZ(stringIdBit5, "db"), "stanza is set"); TEST_RESULT_INT(cfgOptionSource(cfgOptStanza), cfgSourceParam, "stanza is source param"); TEST_RESULT_STR_Z(cfgOptionIdxStr(cfgOptPgPath, 0), "/path/to/db", "pg1-path is set"); TEST_RESULT_INT(cfgOptionSource(cfgOptPgPath), cfgSourceParam, "pg1-path is source param"); - TEST_RESULT_UINT(cfgOptionIdxStrId(cfgOptRepoType, 0), strIdFromZ(stringIdBit6, "s3"), "repo-type is set"); + TEST_RESULT_UINT(cfgOptionIdxStrId(cfgOptRepoType, 0), strIdFromZ("s3"), "repo-type is set"); TEST_RESULT_STR_Z(cfgOptionStr(cfgOptRepoS3Bucket), " test ", "repo1-s3-bucket is set and preserves spaces"); TEST_RESULT_STR_Z(cfgOptionStr(cfgOptRepoS3KeySecret), "xxx", "repo1-s3-secret is set"); TEST_RESULT_INT(cfgOptionSource(cfgOptRepoS3KeySecret), cfgSourceConfig, "repo1-s3-secret is source env"); @@ -1642,6 +1697,9 @@ TEST_RESULT_INT(cfgOptionSource(cfgOptPgPath), cfgSourceConfig, "pg1-path is source config"); TEST_RESULT_STR_Z( cfgOptionIdxStr(cfgOptPgPath, cfgOptionKeyToIdx(cfgOptPgPath, 256)), "/path/to/db256", "pg256-path is set"); + TEST_RESULT_UINT(varUInt64(cfgOptionVar(cfgOptType)), STRID5("incr", 0x90dc90), "check type"); + TEST_RESULT_STR_Z( + cfgOptionDisplayVar(VARUINT64(STRID5("incr", 0x90dc90)), cfgOptTypeStringId), "incr", "check type display"); TEST_RESULT_STR_Z(cfgOptionStr(cfgOptLockPath), "/", "lock-path is set"); TEST_RESULT_INT(cfgOptionSource(cfgOptLockPath), cfgSourceConfig, "lock-path is source config"); TEST_RESULT_STR_Z(cfgOptionIdxStr(cfgOptPgSocketPath, 0), "/path/to/socket", "pg1-socket-path is set"); @@ -1767,6 +1825,35 @@ "key/value 'a' not valid for 'recovery-option' option"); // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("option cmd"); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + strLstAddZ(argList, TEST_COMMAND_BACKUP); + hrnCfgArgRawZ(argList, cfgOptStanza, "db"); + hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, "/path/to/1"); + TEST_RESULT_VOID(configParse(storageTest, strLstSize(argList), strLstPtr(argList), true), "load local config"); + + TEST_RESULT_STR_Z(cfgExe(), "pgbackrest", "--cmd not provided; cfgExe() returns " TEST_BACKREST_EXE); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "db"); + hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, "/path/to/1"); + HRN_CFG_LOAD(cfgCmdRestore, argList); + + TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCmd), TEST_PROJECT_EXE , "--cmd not provided; cmd is defaulted to " TEST_PROJECT_EXE); + + argList = strLstNew(); + strLstAddZ(argList, TEST_BACKREST_EXE); + strLstAddZ(argList, TEST_COMMAND_BACKUP); + hrnCfgArgRawZ(argList, cfgOptStanza, "db"); + hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, "/path/to/1"); + hrnCfgArgRawZ(argList, cfgOptCmd, "pgbackrest_wrapper.sh"); + TEST_RESULT_VOID(configParse(storageTest, strLstSize(argList), strLstPtr(argList), true), "load local config"); + + TEST_RESULT_STR_Z(cfgOptionStr(cfgOptCmd), "pgbackrest_wrapper.sh", "--cmd provided; cmd is returned as pgbackrest_wrapper.sh"); + + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("default job retry and valid duplicate options"); argList = strLstNew(); @@ -1883,6 +1970,12 @@ TEST_RESULT_VOID(cfgOptionIdxSet(cfgOptPgPath, 0, cfgSourceParam, VARSTRDEF("/new")), "set pg1-path"); TEST_RESULT_STR_Z(cfgOptionIdxStr(cfgOptPgPath, 0), "/new", "check pg1-path"); + TEST_RESULT_VOID(cfgOptionIdxSet(cfgOptType, 0, cfgSourceParam, VARUINT64(STRID5("preserve", 0x2da45996500))), "set type"); + TEST_RESULT_UINT(cfgOptionIdxStrId(cfgOptType, 0), STRID5("preserve", 0x2da45996500), "check type"); + + TEST_RESULT_VOID(cfgOptionIdxSet(cfgOptType, 0, cfgSourceParam, VARSTRDEF("standby")), "set type"); + TEST_RESULT_UINT(cfgOptionIdxStrId(cfgOptType, 0), STRID5("standby", 0x6444706930), "check type"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("stanza options should not be loaded for commands that don't take a stanza"); diff -Nru pgbackrest-2.36/test/src/module/db/dbTest.c pgbackrest-2.37/test/src/module/db/dbTest.c --- pgbackrest-2.36/test/src/module/db/dbTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/db/dbTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -4,10 +4,12 @@ #include "common/io/fdRead.h" #include "common/io/fdWrite.h" #include "common/type/json.h" +#include "storage/remote/protocol.h" #include "common/harnessConfig.h" #include "common/harnessFork.h" #include "common/harnessLog.h" +#include "common/harnessPostgres.h" #include "common/harnessPq.h" /*********************************************************************************************************************************** @@ -70,6 +72,7 @@ hrnCfgArgKeyRawZ(argList, cfgOptPgDatabase, 1, "testdb"); hrnCfgArgRawStrId(argList, cfgOptRemoteType, protocolStorageTypePg); hrnCfgArgRawZ(argList, cfgOptProcess, "0"); + hrnCfgArgRawZ(argList, cfgOptDbTimeout, "777"); HRN_CFG_LOAD(cfgCmdBackup, argList, .role = cfgCmdRoleRemote); // Set script @@ -99,7 +102,12 @@ protocolServerNew(STRDEF("db test server"), STRDEF("test"), HRN_FORK_CHILD_READ(), HRN_FORK_CHILD_WRITE()), "create server"); - static const ProtocolServerHandler commandHandler[] = {PROTOCOL_SERVER_HANDLER_DB_LIST}; + static const ProtocolServerHandler commandHandler[] = + { + PROTOCOL_SERVER_HANDLER_DB_LIST + PROTOCOL_SERVER_HANDLER_OPTION_LIST + PROTOCOL_SERVER_HANDLER_STORAGE_REMOTE_LIST + }; TEST_RESULT_VOID( protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)), @@ -110,6 +118,17 @@ HRN_FORK_PARENT_BEGIN() { + // Set options + StringList *argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, TEST_PATH "/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptPgDatabase, 1, "testdb"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 1, "1"); + HRN_CFG_LOAD(cfgCmdBackup, argList); + + // Create control file + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93); + // Create client ProtocolClient *client = NULL; Db *db = NULL; @@ -124,7 +143,7 @@ // ------------------------------------------------------------------------------------------------------------- TEST_TITLE("open and free database"); - TEST_ASSIGN(db, dbNew(NULL, client, STRDEF("test")), "create db"); + TEST_ASSIGN(db, dbNew(NULL, client, storagePgIdx(0), STRDEF("test")), "create db"); TRY_BEGIN() { @@ -143,13 +162,14 @@ // ------------------------------------------------------------------------------------------------------------- TEST_TITLE("remote commands"); - TEST_ASSIGN(db, dbNew(NULL, client, STRDEF("test")), "create db"); + TEST_ASSIGN(db, dbNew(NULL, client, storagePgIdx(0), STRDEF("test")), "create db"); TRY_BEGIN() { TEST_RESULT_VOID(dbOpen(db), "open db"); TEST_RESULT_UINT(db->remoteIdx, 1, "check idx"); TEST_RESULT_STR_Z(dbWalSwitch(db), "000000030000000200000003", "wal switch"); + TEST_RESULT_UINT(dbDbTimeout(db), 777000, "check timeout"); TEST_RESULT_VOID(memContextCallbackClear(db->pub.memContext), "clear context so close is not called"); } FINALLY() @@ -179,8 +199,12 @@ hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 1, "1"); hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, TEST_PATH "/pg1"); hrnCfgArgKeyRawZ(argList, cfgOptPgDatabase, 1, "backupdb"); + hrnCfgArgRawZ(argList, cfgOptDbTimeout, "888"); HRN_CFG_LOAD(cfgCmdBackup, argList); + // Create control file + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93, .checkpoint = pgLsnFromStr(STRDEF("1/1"))); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error when unable to select any pg_settings"); @@ -196,23 +220,26 @@ "[\"select (select setting from pg_catalog.pg_settings where name = 'server_version_num')::int4," " (select setting from pg_catalog.pg_settings where name = 'data_directory')::text," " (select setting from pg_catalog.pg_settings where name = 'archive_mode')::text," - " (select setting from pg_catalog.pg_settings where name = 'archive_command')::text\"]", + " (select setting from pg_catalog.pg_settings where name = 'archive_command')::text," + " (select setting from pg_catalog.pg_settings where name = 'checkpoint_timeout')::int4\"]", .resultInt = 1}, {.session = 1, .function = HRNPQ_CONSUMEINPUT}, {.session = 1, .function = HRNPQ_ISBUSY}, {.session = 1, .function = HRNPQ_GETRESULT}, {.session = 1, .function = HRNPQ_RESULTSTATUS, .resultInt = PGRES_TUPLES_OK}, {.session = 1, .function = HRNPQ_NTUPLES, .resultInt = 1}, - {.session = 1, .function = HRNPQ_NFIELDS, .resultInt = 4}, + {.session = 1, .function = HRNPQ_NFIELDS, .resultInt = 5}, {.session = 1, .function = HRNPQ_FTYPE, .param = "[0]", .resultInt = HRNPQ_TYPE_INT}, {.session = 1, .function = HRNPQ_FTYPE, .param = "[1]", .resultInt = HRNPQ_TYPE_TEXT}, {.session = 1, .function = HRNPQ_FTYPE, .param = "[2]", .resultInt = HRNPQ_TYPE_TEXT}, {.session = 1, .function = HRNPQ_FTYPE, .param = "[3]", .resultInt = HRNPQ_TYPE_TEXT}, + {.session = 1, .function = HRNPQ_FTYPE, .param = "[4]", .resultInt = HRNPQ_TYPE_INT}, {.session = 1, .function = HRNPQ_GETVALUE, .param = "[0,0]", .resultZ = "0"}, {.session = 1, .function = HRNPQ_GETVALUE, .param = "[0,1]", .resultZ = "value"}, {.session = 1, .function = HRNPQ_GETVALUE, .param = "[0,2]", .resultZ = "value"}, {.session = 1, .function = HRNPQ_GETVALUE, .param = "[0,3]", .resultZ = ""}, {.session = 1, .function = HRNPQ_GETISNULL, .param = "[0,3]", .resultInt = 1}, + {.session = 1, .function = HRNPQ_GETVALUE, .param = "[0,4]", .resultZ = "300"}, {.session = 1, .function = HRNPQ_CLEAR}, {.session = 1, .function = HRNPQ_GETRESULT, .resultNull = true}, @@ -241,8 +268,12 @@ HRNPQ_MACRO_ADVISORY_LOCK(1, true), // Start backup with no start fast + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, "000000010000000100000001"), HRNPQ_MACRO_START_BACKUP_83(1, "1/1", "000000010000000100000001"), + // Ping + HRNPQ_MACRO_TIME_QUERY(1, 0), + // Close primary HRNPQ_MACRO_CLOSE(1), @@ -252,13 +283,23 @@ DbGetResult db = {0}; TEST_ASSIGN(db, dbGet(true, true, false), "get primary"); - TEST_RESULT_STR_Z(dbBackupStart(db.primary, false, false).lsn, "1/1", "start backup"); + TEST_RESULT_UINT(dbDbTimeout(db.primary), 888000, "check timeout"); + TEST_RESULT_UINT(dbPgControl(db.primary).timeline, 1, "check timeline"); + + DbBackupStartResult backupStartResult = {0}; + TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, false, true), "start backup"); + TEST_RESULT_STR_Z(backupStartResult.lsn, "1/1", "start backup"); + TEST_RESULT_PTR(backupStartResult.walSegmentCheck, NULL, "WAL segment check"); + + TEST_RESULT_VOID(dbPing(db.primary, false), "ping cluster"); TEST_RESULT_VOID(dbFree(db.primary), "free primary"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("PostgreSQL 9.5 start/stop backup"); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93, .checkpoint = pgLsnFromStr(STRDEF("2/3"))); + harnessPqScriptSet((HarnessPq []) { // Connect to primary @@ -291,12 +332,11 @@ TEST_RESULT_UINT(dbTimeMSec(db.primary), 1000, "check time"); TEST_ERROR( - dbBackupStart(db.primary, false, false), LockAcquireError, + dbBackupStart(db.primary, false, false, false), LockAcquireError, "unable to acquire pgBackRest advisory lock\n" "HINT: is another pgBackRest backup already running on this cluster?"); - DbBackupStartResult backupStartResult = {.lsn = NULL}; - TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, true), "start backup"); + TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, true, false), "start backup"); TEST_RESULT_STR_Z(backupStartResult.lsn, "2/3", "check lsn"); TEST_RESULT_STR_Z(backupStartResult.walSegmentName, "000000010000000200000003", "check wal segment name"); @@ -315,6 +355,8 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("PostgreSQL 9.5 start/stop backup where backup is in progress"); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93, .checkpoint = pgLsnFromStr(STRDEF("2/5"))); + harnessPqScriptSet((HarnessPq []) { // Connect to primary @@ -341,7 +383,7 @@ TEST_ASSIGN(db, dbGet(true, true, false), "get primary"); - TEST_RESULT_STR_Z(dbBackupStart(db.primary, true, true).lsn, "2/5", "start backup"); + TEST_RESULT_STR_Z(dbBackupStart(db.primary, true, true, false).lsn, "2/5", "start backup"); TEST_RESULT_LOG( "P00 WARN: the cluster is already in backup mode but no pgBackRest backup process is running." @@ -354,13 +396,26 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("PostgreSQL 9.6 start/stop backup"); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93, .checkpoint = pgLsnFromStr(STRDEF("3/3"))); + harnessPqScriptSet((HarnessPq []) { // Connect to primary HRNPQ_MACRO_OPEN_GE_96(1, "dbname='backupdb' port=5432", PG_VERSION_96, TEST_PATH "/pg1", false, NULL, NULL), + // Start backup with timeline error + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, "000000020000000300000002"), + HRNPQ_MACRO_START_BACKUP_96(1, false, "3/3", "000000020000000300000003"), + + // Start backup with checkpoint error + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, "000000010000000400000003"), + HRNPQ_MACRO_START_BACKUP_96(1, false, "4/4", "000000010000000400000004"), + // Start backup HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_LE_96(1, "000000010000000300000002"), HRNPQ_MACRO_START_BACKUP_96(1, false, "3/3", "000000010000000300000003"), // Stop backup @@ -374,9 +429,16 @@ TEST_ASSIGN(db, dbGet(true, true, false), "get primary"); - TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, true), "start backup"); + TEST_ERROR( + dbBackupStart(db.primary, false, true, true), DbMismatchError, "WAL timeline 2 does not match pg_control timeline 1"); + TEST_ERROR( + dbBackupStart(db.primary, false, true, true), DbMismatchError, + "current checkpoint '3/3' is less than backup start '4/4'"); + + TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, true, true), "start backup"); TEST_RESULT_STR_Z(backupStartResult.lsn, "3/3", "check lsn"); TEST_RESULT_STR_Z(backupStartResult.walSegmentName, "000000010000000300000003", "check wal segment name"); + TEST_RESULT_STR_Z(backupStartResult.walSegmentCheck, "000000010000000300000002", "check wal segment check"); TEST_ASSIGN(backupStopResult, dbBackupStop(db.primary), "stop backup"); TEST_RESULT_STR_Z(backupStopResult.lsn, "3/4", "check lsn"); @@ -397,6 +459,10 @@ hrnCfgArgKeyRawZ(argList, cfgOptPgPort, 2, "5433"); HRN_CFG_LOAD(cfgCmdBackup, argList); + // Create control file + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93, .timeline = 5, .checkpoint = pgLsnFromStr(STRDEF("5/4"))); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_93, .timeline = 5, .checkpoint = pgLsnFromStr(STRDEF("5/4"))); + harnessPqScriptSet((HarnessPq []) { // Connect to primary @@ -412,6 +478,16 @@ // Wait for standby to sync HRNPQ_MACRO_REPLAY_WAIT_LE_95(2, "5/4"), + // Ping + HRNPQ_MACRO_IS_STANDBY_QUERY(1, true), + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(1, false), + + HRNPQ_MACRO_IS_STANDBY_QUERY(2, false), + HRNPQ_MACRO_IS_STANDBY_QUERY(2, true), + HRNPQ_MACRO_IS_STANDBY_QUERY(2, true), + HRNPQ_MACRO_IS_STANDBY_QUERY(2, true), + // Close standby HRNPQ_MACRO_CLOSE(2), @@ -423,8 +499,24 @@ TEST_ASSIGN(db, dbGet(false, true, true), "get primary and standby"); - TEST_RESULT_STR_Z(dbBackupStart(db.primary, false, false).lsn, "5/4", "start backup"); - TEST_RESULT_VOID(dbReplayWait(db.standby, STRDEF("5/4"), 1000), "sync standby"); + TEST_RESULT_STR_Z(dbBackupStart(db.primary, false, false, false).lsn, "5/4", "start backup"); + TEST_RESULT_VOID(dbReplayWait(db.standby, STRDEF("5/4"), dbPgControl(db.primary).timeline, 1000), "sync standby"); + + TEST_ERROR(dbPing(db.primary, false), AssertError, "primary has switched to recovery"); + TEST_RESULT_VOID(dbPing(db.primary, false), "ping primary cluster"); + TEST_RESULT_VOID(dbPing(db.primary, false), "ping primary cluster (noop)"); + TEST_RESULT_VOID(dbPing(db.primary, true), "ping primary cluster (force)"); + + TEST_ERROR( + dbPing(db.standby, false), DbMismatchError, + "standby is no longer is recovery\n" + "HINT: was the standby promoted during the backup?"); + TEST_RESULT_VOID(dbPing(db.standby, false), "ping standby cluster"); + TEST_RESULT_VOID(dbPing(db.standby, false), "ping standby cluster (noop)"); + TEST_RESULT_VOID(dbPing(db.standby, true), "ping standby cluster (force)"); + + db.standby->pingTimeLast -= DB_PING_SEC * 2; + TEST_RESULT_VOID(dbPing(db.standby, false), "ping standby cluster"); TEST_RESULT_VOID(dbFree(db.standby), "free standby"); TEST_RESULT_VOID(dbFree(db.primary), "free primary"); @@ -432,6 +524,10 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("PostgreSQL 10 start/stop backup from standby"); + // Update control file + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93, .timeline = 5, .checkpoint = pgLsnFromStr(STRDEF("5/5"))); + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_93, .timeline = 5, .checkpoint = pgLsnFromStr(STRDEF("5/5"))); + harnessPqScriptSet((HarnessPq []) { // Connect to primary @@ -442,8 +538,13 @@ // Start backup HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_GE_10(1, "000000050000000500000005"), HRNPQ_MACRO_START_BACKUP_GE_10(1, false, "5/5", "000000050000000500000005"), + // Switch WAL segment so it can be checked + HRNPQ_MACRO_CREATE_RESTORE_POINT(1, "5/5"), + HRNPQ_MACRO_WAL_SWITCH(1, "wal", "000000050000000500000005"), + // Standby returns NULL lsn {.session = 2, .function = HRNPQ_SENDQUERY, @@ -499,30 +600,78 @@ TEST_ASSIGN(db, dbGet(false, true, true), "get primary and standby"); - TEST_RESULT_STR_Z(dbBackupStart(db.primary, false, false).lsn, "5/5", "start backup"); + TEST_RESULT_UINT(dbPgControl(db.primary).timeline, 5, "check primary timeline"); + TEST_RESULT_UINT(dbPgControl(db.standby).timeline, 5, "check standby timeline"); + + TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, false, true), "start backup"); + TEST_RESULT_STR_Z(backupStartResult.lsn, "5/5", "check lsn"); + TEST_RESULT_STR_Z(backupStartResult.walSegmentName, "000000050000000500000005", "check wal segment name"); + TEST_RESULT_STR_Z(backupStartResult.walSegmentCheck, "000000050000000500000005", "check wal segment check"); TEST_ERROR( - dbReplayWait(db.standby, STRDEF("5/5"), 1000), ArchiveTimeoutError, + dbReplayWait(db.standby, STRDEF("5/5"), 77, 1000), DbMismatchError, "standby is on timeline 5 but expected 77"); + TEST_ERROR( + dbReplayWait(db.standby, STRDEF("4/4"), 5, 1000), DbMismatchError, "standby checkpoint '5/5' is ahead of target '4/4'"); + + TEST_ERROR( + dbReplayWait(db.standby, STRDEF("5/5"), dbPgControl(db.primary).timeline, 1000), ArchiveTimeoutError, "unable to query replay lsn on the standby using 'pg_catalog.pg_last_wal_replay_lsn()'\n" "HINT: Is this a standby?"); TEST_ERROR( - dbReplayWait(db.standby, STRDEF("5/5"), 200), ArchiveTimeoutError, + dbReplayWait(db.standby, STRDEF("5/5"), dbPgControl(db.primary).timeline, 200), ArchiveTimeoutError, "timeout before standby replayed to 5/5 - only reached 5/3\n" "HINT: is replication running and current on the standby?\n" "HINT: disable the 'backup-standby' option to backup directly from the primary."); TEST_ERROR( - dbReplayWait(db.standby, STRDEF("5/5"), 200), ArchiveTimeoutError, + dbReplayWait(db.standby, STRDEF("5/5"), dbPgControl(db.primary).timeline, 200), ArchiveTimeoutError, "timeout before standby checkpoint lsn reached 5/5 - only reached 5/4"); - TEST_RESULT_VOID(dbReplayWait(db.standby, STRDEF("5/5"), 1000), "sync standby"); + TEST_RESULT_VOID(dbReplayWait(db.standby, STRDEF("5/5"), dbPgControl(db.primary).timeline, 1000), "sync standby"); TEST_RESULT_VOID(dbFree(db.standby), "free standby"); TEST_RESULT_STR_Z(dbBackupStop(db.primary).tablespaceMap, "TABLESPACE_MAP_DATA", "stop backup"); TEST_RESULT_VOID(dbFree(db.primary), "free primary"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("PostgreSQL 14 - checkpoint timeout warning"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 1, "1"); + hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, TEST_PATH "/pg1"); + // With start-fast being disabled, set db-timeout smaller than checkpoint_timeout to raise a warning + hrnCfgArgRawZ(argList, cfgOptDbTimeout, "299"); + HRN_CFG_LOAD(cfgCmdBackup, argList); + + harnessPqScriptSet((HarnessPq []) + { + // Connect to primary + HRNPQ_MACRO_OPEN_GE_96(1, "dbname='postgres' port=5432", PG_VERSION_14, TEST_PATH "/pg1", false, NULL, NULL), + + // Start backup + HRNPQ_MACRO_ADVISORY_LOCK(1, true), + HRNPQ_MACRO_CURRENT_WAL_GE_10(1, "000000050000000500000004"), + HRNPQ_MACRO_START_BACKUP_GE_10(1, false, "5/5", "000000050000000500000005"), + + // Close primary + HRNPQ_MACRO_CLOSE(1), + + HRNPQ_MACRO_DONE() + }); + + TEST_ASSIGN(db, dbGet(true, true, false), "get primary"); + TEST_ASSIGN(backupStartResult, dbBackupStart(db.primary, false, false, true), "start backup"); + + TEST_RESULT_LOG( + "P00 WARN: start-fast is disabled and db-timeout (299s) is smaller than the PostgreSQL checkpoint_timeout (300s) -" + " timeout may occur before the backup starts"); + + TEST_RESULT_VOID(dbFree(db.primary), "free primary"); + } // ***************************************************************************************************************************** @@ -537,6 +686,9 @@ hrnCfgArgKeyRawZ(argList, cfgOptPgUser, 1, "bob"); HRN_CFG_LOAD(cfgCmdBackup, argList); + // Create control file + HRN_PG_CONTROL_PUT(storagePgIdxWrite(0), PG_VERSION_93); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error connecting to primary"); @@ -620,6 +772,9 @@ hrnCfgArgKeyRawZ(argList, cfgOptPgPort, 8, "5433"); HRN_CFG_LOAD(cfgCmdBackup, argList); + // Create control file + HRN_PG_CONTROL_PUT(storagePgIdxWrite(1), PG_VERSION_93); + harnessPqScriptSet((HarnessPq []) { HRNPQ_MACRO_OPEN_LE_91(1, "dbname='postgres' port=5432", PG_VERSION_84, TEST_PATH "/pg1", NULL, NULL), diff -Nru pgbackrest-2.36/test/src/module/info/infoBackupTest.c pgbackrest-2.37/test/src/module/info/infoBackupTest.c --- pgbackrest-2.36/test/src/module/info/infoBackupTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/info/infoBackupTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -190,7 +190,7 @@ (strLstSize(backupDataPtr->backupReference) == 1 && strLstExists(backupDataPtr->backupReference, STRDEF("20161219-212741F"))), true, "backup reference exists"); - TEST_RESULT_PTR(infoBackupDataByLabel(infoBackup, STRDEF("20161219-12345")), NULL, "backup label does not exist"); + TEST_RESULT_BOOL(infoBackupLabelExists(infoBackup, STRDEF("20161219-12345")), false, "backup label does not exist"); TEST_RESULT_BOOL(varBool(backupDataPtr->backupError), true, "backup error"); backupData = infoBackupData(infoBackup, 2); diff -Nru pgbackrest-2.36/test/src/module/info/infoPgTest.c pgbackrest-2.37/test/src/module/info/infoPgTest.c --- pgbackrest-2.36/test/src/module/info/infoPgTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/info/infoPgTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -191,13 +191,13 @@ //-------------------------------------------------------------------------------------------------------------------------- pgData.id = 3; pgData.version = PG_VERSION_96; - pgData.systemId = 6399999999999999999; + pgData.systemId = HRN_PG_SYSTEMID_96; TEST_RESULT_VOID(infoPgAdd(infoPg, &pgData), "infoPgAdd"); InfoPgData pgDataTest = infoPgDataCurrent(infoPg); TEST_RESULT_INT(pgDataTest.id, 3, " id set"); TEST_RESULT_INT(pgDataTest.version, PG_VERSION_96, " version set"); - TEST_RESULT_UINT(pgDataTest.systemId, 6399999999999999999, " system-id set"); + TEST_RESULT_UINT(pgDataTest.systemId, HRN_PG_SYSTEMID_96, " system-id set"); // infoPgDataToLog //-------------------------------------------------------------------------------------------------------------------------- diff -Nru pgbackrest-2.36/test/src/module/performance/storageTest.c pgbackrest-2.37/test/src/module/performance/storageTest.c --- pgbackrest-2.36/test/src/module/performance/storageTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/performance/storageTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -149,7 +149,7 @@ TEST_TITLE_FMT("list %d million files", TEST_SCALE); // One million files represents a fairly large cluster - CHECK(TEST_SCALE <= 2000); + ASSERT(TEST_SCALE <= 2000); uint64_t fileTotal = (uint64_t)1000000 * TEST_SCALE; HRN_FORK_BEGIN(.timeout = 60000) @@ -172,8 +172,7 @@ driver.interface.infoList = storageTestPerfInfoList; - Storage *storageTest = storageNew( - strIdFromZ(stringIdBit6, "test"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface); + Storage *storageTest = storageNew(strIdFromZ("test"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface); storageHelper.storageRepoWrite = memNew(sizeof(Storage *)); storageHelper.storageRepoWrite[0] = storageTest; @@ -226,7 +225,7 @@ ioBufferSizeSet(4 * 1024 * 1024); // 1MB is a fairly normal table size - CHECK(TEST_SCALE <= 1024 * 1024 * 1024); + ASSERT(TEST_SCALE <= 1024 * 1024 * 1024); uint64_t blockTotal = (uint64_t)1 * TEST_SCALE; // Set iteration diff -Nru pgbackrest-2.36/test/src/module/performance/typeTest.c pgbackrest-2.37/test/src/module/performance/typeTest.c --- pgbackrest-2.36/test/src/module/performance/typeTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/performance/typeTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -8,15 +8,19 @@ running out of memory on the test systems or taking an undue amount of time. It should be noted that in this context scaling to 1000 is nowhere near turning it up to 11. ***********************************************************************************************************************************/ +#include + #include "common/ini.h" #include "common/io/bufferRead.h" #include "common/io/bufferWrite.h" +#include "common/io/socket/client.h" #include "common/stat.h" #include "common/time.h" #include "common/type/list.h" #include "common/type/object.h" #include "info/manifest.h" #include "postgres/version.h" +#include "storage/posix/storage.h" #include "common/harnessInfo.h" #include "common/harnessStorage.h" @@ -155,7 +159,7 @@ // ***************************************************************************************************************************** if (testBegin("lstFind()")) { - CHECK(TEST_SCALE <= 10000); + ASSERT(TEST_SCALE <= 10000); int testMax = 100000 * (int)TEST_SCALE; // Generate a large list of values (use int instead of string so there fewer allocations) @@ -164,7 +168,7 @@ for (int listIdx = 0; listIdx < testMax; listIdx++) lstAdd(list, &listIdx); - CHECK(lstSize(list) == (unsigned int)testMax); + ASSERT(lstSize(list) == (unsigned int)testMax); TEST_LOG_FMT("generated %d item list", testMax); @@ -174,7 +178,7 @@ TimeMSec timeBegin = timeMSec(); for (int listIdx = 0; listIdx < testMax; listIdx++) - CHECK(*(int *)lstFind(list, &listIdx) == listIdx); + ASSERT(*(int *)lstFind(list, &listIdx) == listIdx); TEST_LOG_FMT("asc search completed in %ums", (unsigned int)(timeMSec() - timeBegin)); @@ -184,7 +188,7 @@ timeBegin = timeMSec(); for (int listIdx = 0; listIdx < testMax; listIdx++) - CHECK(*(int *)lstFind(list, &listIdx) == listIdx); + ASSERT(*(int *)lstFind(list, &listIdx) == listIdx); TEST_LOG_FMT("desc search completed in %ums", (unsigned int)(timeMSec() - timeBegin)); } @@ -192,7 +196,7 @@ // ***************************************************************************************************************************** if (testBegin("lstRemoveIdx()")) { - CHECK(TEST_SCALE <= 10000); + ASSERT(TEST_SCALE <= 10000); int testMax = 1000000 * (int)TEST_SCALE; // Generate a large list of values (use int instead of string so there fewer allocations) @@ -201,7 +205,7 @@ for (int listIdx = 0; listIdx < testMax; listIdx++) lstAdd(list, &listIdx); - CHECK(lstSize(list) == (unsigned int)testMax); + ASSERT(lstSize(list) == (unsigned int)testMax); TEST_LOG_FMT("generated %d item list", testMax); @@ -213,13 +217,13 @@ TEST_LOG_FMT("remove completed in %ums", (unsigned int)(timeMSec() - timeBegin)); - CHECK(lstEmpty(list)); + ASSERT(lstEmpty(list)); } // ***************************************************************************************************************************** if (testBegin("iniLoad()")) { - CHECK(TEST_SCALE <= 10000); + ASSERT(TEST_SCALE <= 10000); String *iniStr = strCatZ(strNew(), "[section1]\n"); unsigned int iniMax = 100000 * (unsigned int)TEST_SCALE; @@ -242,7 +246,7 @@ // ***************************************************************************************************************************** if (testBegin("manifestNewBuild()/manifestNewLoad()/manifestSave()")) { - CHECK(TEST_SCALE <= 1000000); + ASSERT(TEST_SCALE <= 1000000); // Create a storage driver to test manifest build with an arbitrary number of files StorageTestManifestNewBuild driver = @@ -255,7 +259,7 @@ driver.interface.infoList = storageTestManifestNewBuildInfoList; const Storage *const storagePg = storageNew( - strIdFromZ(stringIdBit6, "test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface); + strIdFromZ("test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("build manifest"); @@ -314,7 +318,7 @@ for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { const ManifestFile *file = manifestFile(manifest, fileIdx); - CHECK(file == manifestFileFind(manifest, file->name)); + ASSERT(file == manifestFileFind(manifest, file->name)); } TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin)); @@ -324,7 +328,7 @@ // ***************************************************************************************************************************** if (testBegin("statistics collector")) { - CHECK(TEST_SCALE <= 1000000); + ASSERT(TEST_SCALE <= 1000000); // Setup a list of stats to use for testing #define TEST_STAT_TOTAL 100 @@ -361,5 +365,24 @@ } } + // ***************************************************************************************************************************** + if (testBegin("SocketClient")) + { + // This test must be done here because the problem with variables being clobbered after a long jump is only present in + // optimized builds, so the unit test will not notice if the volatile keyword goes missing in sckClientOpen(). Since the + // performance tests are built with optimization is it more likely to be caught here. + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("create socket with error to check for leaks"); + + const Storage *const storageFd = storagePosixNewP(strNewFmt("/proc/%d/fd", getpid())); + unsigned int fdBefore = strLstSize(storageListP(storageFd, NULL)); + + TEST_ERROR( + ioClientOpen(sckClientNew(STRDEF("172.31.255.255"), 7777, 0, 0)), HostConnectError, + "timeout connecting to '172.31.255.255:7777'"); + + TEST_RESULT_UINT(strLstSize(storageListP(storageFd, NULL)), fdBefore, "socket was freed"); + } + FUNCTION_HARNESS_RETURN_VOID(); } diff -Nru pgbackrest-2.36/test/src/module/postgres/interfaceTest.c pgbackrest-2.37/test/src/module/postgres/interfaceTest.c --- pgbackrest-2.36/test/src/module/postgres/interfaceTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/postgres/interfaceTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -47,8 +47,6 @@ TEST_RESULT_UINT(pgInterface[0].version, PG_VERSION_MAX, "check max version"); //-------------------------------------------------------------------------------------------------------------------------- - const String *controlFile = STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL); - // Create a bogus control file Buffer *result = bufNew(HRN_PG_CONTROL_SIZE); memset(bufPtr(result), 0, bufSize(result)); @@ -65,46 +63,44 @@ "unexpected control version = 501 and catalog version = 19780101\nHINT: is this version of PostgreSQL supported?"); //-------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageTest, controlFile), - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACE, .walSegmentSize = 1024 * 1024})); + HRN_PG_CONTROL_PUT( + storageTest, PG_VERSION_11, .systemId = 0xFACEFACE, .checkpoint = 0xEEFFEEFFAABBAABB, .timeline = 47, + .walSegmentSize = 1024 * 1024); PgControl info = {0}; TEST_ASSIGN(info, pgControlFromFile(storageTest), "get control info v11"); TEST_RESULT_UINT(info.systemId, 0xFACEFACE, " check system id"); TEST_RESULT_UINT(info.version, PG_VERSION_11, " check version"); TEST_RESULT_UINT(info.catalogVersion, 201809051, " check catalog version"); + TEST_RESULT_UINT(info.checkpoint, 0xEEFFEEFFAABBAABB, "check checkpoint"); + TEST_RESULT_UINT(info.timeline, 47, "check timeline"); //-------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageTest, controlFile), - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_93, .walSegmentSize = 1024 * 1024})); + HRN_PG_CONTROL_PUT(storageTest, PG_VERSION_93, .walSegmentSize = 1024 * 1024); TEST_ERROR( pgControlFromFile(storageTest), FormatError, "wal segment size is 1048576 but must be 16777216 for PostgreSQL <= 10"); //-------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageTest, controlFile), - hrnPgControlToBuffer((PgControl){.version = PG_VERSION_95, .pageSize = 32 * 1024})); + HRN_PG_CONTROL_PUT(storageTest, PG_VERSION_95, .pageSize = 32 * 1024); TEST_ERROR(pgControlFromFile(storageTest), FormatError, "page size is 32768 but must be 8192"); //-------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageTest, controlFile), - hrnPgControlToBuffer( - (PgControl){ - .version = PG_VERSION_83, .systemId = 0xEFEFEFEFEF, .catalogVersion = hrnPgCatalogVersion(PG_VERSION_83)})); + HRN_PG_CONTROL_PUT( + storageTest, PG_VERSION_83, .systemId = 0xEFEFEFEFEF, .catalogVersion = hrnPgCatalogVersion(PG_VERSION_83), + .checkpoint = 0xAABBAABBEEFFEEFF, .timeline = 88); TEST_ASSIGN(info, pgControlFromFile(storageTest), "get control info v83"); TEST_RESULT_UINT(info.systemId, 0xEFEFEFEFEF, " check system id"); TEST_RESULT_UINT(info.version, PG_VERSION_83, " check version"); TEST_RESULT_UINT(info.catalogVersion, 200711281, " check catalog version"); + TEST_RESULT_UINT(info.checkpoint, 0xAABBAABBEEFFEEFF, "check checkpoint"); + TEST_RESULT_UINT(info.timeline, 88, "check timeline"); } // ***************************************************************************************************************************** - if (testBegin("pgLsnFromStr(), pgLsnToStr(), pgLsnToWalSegment(), pgLsnFromWalSegment(), and pgLsnRangeToWalSegmentList()")) + if (testBegin("pgLsnFromStr(), pgLsnToStr(), pgLsnToWalSegment(), pg*FromWalSegment(), and pgLsnRangeToWalSegmentList()")) { TEST_RESULT_UINT(pgLsnFromStr(STRDEF("1/1")), 0x0000000100000001, "lsn to string"); TEST_RESULT_UINT(pgLsnFromStr(STRDEF("ffffffff/ffffffff")), 0xFFFFFFFFFFFFFFFF, "lsn to string"); @@ -125,6 +121,9 @@ TEST_RESULT_UINT( pgLsnFromWalSegment(STRDEF("00000001FFFFFFFF00000001"), 0x40000000), 0xFFFFFFFF40000000, "1G wal segment to lsn"); + TEST_RESULT_UINT(pgTimelineFromWalSegment(STRDEF("00000001FFFFFFFF000000AA")), 1, "timeline 1"); + TEST_RESULT_UINT(pgTimelineFromWalSegment(STRDEF("F000000FFFFFFFFF000000AA")), 0xF000000F, "timeline F000000F"); + TEST_RESULT_STRLST_Z( pgLsnRangeToWalSegmentList( PG_VERSION_92, 1, pgLsnFromStr(STRDEF("1/60")), pgLsnFromStr(STRDEF("1/60")), 16 * 1024 * 1024), diff -Nru pgbackrest-2.36/test/src/module/protocol/protocolTest.c pgbackrest-2.37/test/src/module/protocol/protocolTest.c --- pgbackrest-2.36/test/src/module/protocol/protocolTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/protocol/protocolTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -594,7 +594,7 @@ TEST_TITLE("invalid command"); TEST_ERROR( - protocolClientExecute(client, protocolCommandNew(strIdFromZ(stringIdBit6, "BOGUS")), false), ProtocolError, + protocolClientExecute(client, protocolCommandNew(strIdFromZ("BOGUS")), false), ProtocolError, "raised from test client: invalid command 'BOGUS' (0x38eacd271)"); // ----------------------------------------------------------------------------------------------------------------- @@ -711,9 +711,8 @@ TEST_TITLE("ping server"); // Connect to server without any verification - IoClient *tlsClient = tlsClientNew( - sckClientNew(hrnServerHost(), hrnServerPort(0), 5000, 5000), hrnServerHost(), 5000, 5000, false, NULL, NULL, - NULL, NULL); + IoClient *tlsClient = tlsClientNewP( + sckClientNew(hrnServerHost(), hrnServerPort(0), 5000, 5000), hrnServerHost(), 5000, 5000, false); IoSession *tlsSession = ioClientOpen(tlsClient); // Send ping @@ -816,7 +815,7 @@ StringList *argList = strLstDup(argListBase); hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "pgbackrest-client=db"); - HRN_CFG_LOAD(cfgCmdServerStart, argList); + HRN_CFG_LOAD(cfgCmdServer, argList); socketSession = ioServerAccept(socketServer, NULL); @@ -828,7 +827,7 @@ // ----------------------------------------------------------------------------------------------------------------- argList = strLstDup(argListBase); hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "pgbackrest-client=bogus"); - HRN_CFG_LOAD(cfgCmdServerStart, argList); + HRN_CFG_LOAD(cfgCmdServer, argList); socketSession = ioServerAccept(socketServer, NULL); @@ -838,7 +837,7 @@ // ----------------------------------------------------------------------------------------------------------------- argList = strLstDup(argListBase); hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "bogus=*"); - HRN_CFG_LOAD(cfgCmdServerStart, argList); + HRN_CFG_LOAD(cfgCmdServer, argList); socketSession = ioServerAccept(socketServer, NULL); @@ -848,7 +847,7 @@ // ----------------------------------------------------------------------------------------------------------------- argList = strLstDup(argListBase); hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "pgbackrest-client=db"); - HRN_CFG_LOAD(cfgCmdServerStart, argList); + HRN_CFG_LOAD(cfgCmdServer, argList); socketSession = ioServerAccept(socketServer, NULL); @@ -858,7 +857,7 @@ // ----------------------------------------------------------------------------------------------------------------- argList = strLstDup(argListBase); hrnCfgArgRawZ(argList, cfgOptTlsServerAuth, "pgbackrest-client=*"); - HRN_CFG_LOAD(cfgCmdServerStart, argList); + HRN_CFG_LOAD(cfgCmdServer, argList); socketSession = ioServerAccept(socketServer, NULL); @@ -883,7 +882,7 @@ { TEST_ASSIGN( job, - protocolParallelJobNew(VARSTRDEF("test"), protocolCommandNew(strIdFromZ(stringIdBit5, "c"))), "new job"); + protocolParallelJobNew(VARSTRDEF("test"), protocolCommandNew(strIdFromZ("c"))), "new job"); TEST_RESULT_PTR(protocolParallelJobMove(job, memContextPrior()), job, "move job"); TEST_RESULT_PTR(protocolParallelJobMove(NULL, memContextPrior()), NULL, "move null job"); } @@ -915,7 +914,7 @@ "local server 1"); // Command with output - TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-one"), "c-one command get"); + TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c-one"), "c-one command get"); // Wait for notify from parent HRN_FORK_CHILD_NOTIFY_GET(); @@ -938,7 +937,7 @@ "local server 2"); // Command with output - TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c2"), "c2 command get"); + TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c2"), "c2 command get"); // Wait for notify from parent HRN_FORK_CHILD_NOTIFY_GET(); @@ -947,11 +946,11 @@ TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put"); // Command with error - TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-three"), "c-three command get"); + TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c-three"), "c-three command get"); TEST_RESULT_VOID(protocolServerError(server, 39, STRDEF("very serious error"), STRDEF("stack")), "error put"); // Wait for exit - CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_EXIT); + TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "wait for exit"); } HRN_FORK_CHILD_END(); @@ -993,20 +992,20 @@ // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("add jobs"); - ProtocolCommand *command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-one")); + ProtocolCommand *command = protocolCommandNew(strIdFromZ("c-one")); pckWriteStrP(protocolCommandParam(command), STRDEF("param1")); pckWriteStrP(protocolCommandParam(command), STRDEF("param2")); ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command); TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job"); - command = protocolCommandNew(strIdFromZ(stringIdBit5, "c2")); + command = protocolCommandNew(strIdFromZ("c2")); pckWriteStrP(protocolCommandParam(command), STRDEF("param1")); job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command); TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job"); - command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-three")); + command = protocolCommandNew(strIdFromZ("c-three")); pckWriteStrP(protocolCommandParam(command), STRDEF("param1")); job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command); diff -Nru pgbackrest-2.36/test/src/module/storage/azureTest.c pgbackrest-2.37/test/src/module/storage/azureTest.c --- pgbackrest-2.36/test/src/module/storage/azureTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/storage/azureTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -206,13 +206,157 @@ // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("storage with host but force host-style uri"); + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "https://test-host"); + hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStyleHost); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".test-host", "check host"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with https protocol, appended port, uristylepath"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "https://test-host:443"); + hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStylePath); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, "test-host", "check host"); + TEST_RESULT_STR_Z( + ((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_ACCOUNT "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with no protocol, appended port, uristylepath"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "test-host:443"); + hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStylePath); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, "test-host", "check host"); + TEST_RESULT_STR_Z( + ((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_ACCOUNT "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with https protocol, appended port, uristylehost"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "https://test-host:443"); + hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStyleHost); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".test-host", "check host"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with no protocol, appended port, uristylehost"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "test-host:443"); + hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStyleHost); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".test-host", "check host"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with no protocol, with appended port, default uristyle "); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "test-host:443"); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, "test-host", "check host"); + TEST_RESULT_STR_Z( + ((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_ACCOUNT "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with no protocol, specified port, uristylehost"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "test-host"); + hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", (const unsigned int) 443); hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStyleHost); HRN_CFG_LOAD(cfgCmdArchivePush, argList); TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".test-host", "check host"); TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("storage with no protocol, appended port, specified port"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test"); + hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); + hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); + hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); + hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); + hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "test-host:8443"); + hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", (const unsigned int) 8443); + HRN_CFG_LOAD(cfgCmdArchivePush, argList); + + TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage"); + TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, "test-host", "check host"); + TEST_RESULT_STR_Z( + ((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_ACCOUNT "/" TEST_CONTAINER, "check path prefix"); } // ***************************************************************************************************************************** @@ -302,8 +446,7 @@ hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE); hrnCfgArgRawZ(argList, cfgOptRepoPath, "/"); hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER); - hrnCfgArgRaw(argList, cfgOptRepoStorageHost, hrnServerHost()); - hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", hrnServerPort(0)); + hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "https://%s:%u", strZ(hrnServerHost()), hrnServerPort(0)); hrnCfgArgRawBool(argList, cfgOptRepoStorageVerifyTls, TEST_IN_CONTAINER); hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT); hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SHARED); @@ -373,7 +516,7 @@ ioReadOpen(storageReadIo(read)), ProtocolError, "HTTP request failed with 303:\n" "*** Path/Query ***:\n" - "/account/container/file.txt\n" + "GET /account/container/file.txt\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" @@ -396,7 +539,7 @@ storagePutP(storageNewWriteP(storage, STRDEF("file.txt")), BUFSTRDEF("ABCD")), ProtocolError, "HTTP request failed with 403 (Forbidden):\n" "*** Path/Query ***:\n" - "/account/container/file.txt\n" + "PUT /account/container/file.txt\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 4\n" @@ -780,7 +923,7 @@ storagePathRemoveP(storage, STRDEF("/"), .recurse = true), ProtocolError, "HTTP request failed with 403 (Forbidden):\n" "*** Path/Query ***:\n" - "/account/container?comp=list&restype=container&sig=\n" + "GET /account/container?comp=list&restype=container&sig=\n" "*** Request Headers ***:\n" "content-length: 0\n" "host: %s", diff -Nru pgbackrest-2.36/test/src/module/storage/gcsTest.c pgbackrest-2.37/test/src/module/storage/gcsTest.c --- pgbackrest-2.36/test/src/module/storage/gcsTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/storage/gcsTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -478,7 +478,7 @@ ioReadOpen(storageReadIo(read)), ProtocolError, "HTTP request failed with 303:\n" "*** Path/Query ***:\n" - "/storage/v1/b/bucket/o/file.txt?alt=media\n" + "GET /storage/v1/b/bucket/o/file.txt?alt=media\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" @@ -501,7 +501,7 @@ storagePutP(storageNewWriteP(storage, STRDEF("file.txt")), BUFSTRDEF("ABCD")), ProtocolError, "HTTP request failed with 403 (Forbidden):\n" "*** Path/Query ***:\n" - "/upload/storage/v1/b/bucket/o?fields=md5Hash%%2Csize&name=file.txt&uploadType=media\n" + "POST /upload/storage/v1/b/bucket/o?fields=md5Hash%%2Csize&name=file.txt&uploadType=media\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 4\n" @@ -634,7 +634,7 @@ storagePutP(storageNewWriteP(storage, STRDEF("file.txt")), BUFSTRDEF("12345678901234567")), ProtocolError, "HTTP request failed with 403 (Forbidden):\n" "*** Path/Query ***:\n" - "/upload/storage/v1/b/bucket/o?name=file.txt&uploadType=resumable&upload_id=\n" + "PUT /upload/storage/v1/b/bucket/o?name=file.txt&uploadType=resumable&upload_id=\n" "*** Request Headers ***:\n" "content-length: 16\n" "content-range: bytes 0-15/*\n" diff -Nru pgbackrest-2.36/test/src/module/storage/posixTest.c pgbackrest-2.37/test/src/module/storage/posixTest.c --- pgbackrest-2.36/test/src/module/storage/posixTest.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/storage/posixTest.c 2022-01-03 13:43:55.000000000 +0000 @@ -1545,7 +1545,7 @@ static const StorageHelper storageHelperListError[] = {{.type = STORAGE_POSIX_TYPE}, STORAGE_END_HELPER}; storageHelperInit(storageHelperListError); - TEST_ERROR(storageRepoGet(0, true), AssertError, "check 'type == STORAGE_POSIX_TYPE' failed"); + TEST_ERROR(storageRepoGet(0, true), AssertError, "invalid storage type"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("storage configuration"); diff -Nru pgbackrest-2.36/test/src/module/storage/s3Test.c pgbackrest-2.37/test/src/module/storage/s3Test.c --- pgbackrest-2.36/test/src/module/storage/s3Test.c 2021-11-01 12:59:14.000000000 +0000 +++ pgbackrest-2.37/test/src/module/storage/s3Test.c 2022-01-03 13:43:55.000000000 +0000 @@ -478,7 +478,7 @@ storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError, "HTTP request failed with 301:\n" "*** Path/Query ***:\n" - "/latest/meta-data/iam/security-credentials\n" + "GET /latest/meta-data/iam/security-credentials\n" "*** Request Headers ***:\n" "content-length: 0\n" "host: %s", @@ -519,7 +519,7 @@ storageGetP(storageNewReadP(s3, STRDEF("file.txt"))), ProtocolError, "HTTP request failed with 300:\n" "*** Path/Query ***:\n" - "/latest/meta-data/iam/security-credentials/credrole\n" + "GET /latest/meta-data/iam/security-credentials/credrole\n" "*** Request Headers ***:\n" "content-length: 0\n" "host: %s", @@ -583,7 +583,7 @@ ioReadOpen(storageReadIo(read)), ProtocolError, "HTTP request failed with 303:\n" "*** Path/Query ***:\n" - "/file.txt\n" + "GET /file.txt\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" @@ -730,7 +730,7 @@ storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), ProtocolError, "HTTP request failed with 200 (OK):\n" "*** Path/Query ***:\n" - "/file.txt?uploadId=WxRt\n" + "POST /file.txt?uploadId=WxRt\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 205\n" @@ -926,7 +926,7 @@ TEST_ERROR(storageListP(s3, STRDEF("/")), ProtocolError, "HTTP request failed with 344:\n" "*** Path/Query ***:\n" - "/?delimiter=%2F&list-type=2\n" + "GET /?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" @@ -950,7 +950,7 @@ TEST_ERROR(storageListP(s3, STRDEF("/")), ProtocolError, "HTTP request failed with 344:\n" "*** Path/Query ***:\n" - "/?delimiter=%2F&list-type=2\n" + "GET /?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n" "authorization: \n" "content-length: 0\n" @@ -1153,7 +1153,7 @@ argList = strLstDup(commonArgList); hrnCfgArgRawStrId(argList, cfgOptRepoS3UriStyle, storageS3UriStylePath); - hrnCfgArgRaw(argList, cfgOptRepoStorageHost, host); + hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "https://%s", strZ(host)); hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", port); hrnCfgEnvRemoveRaw(cfgOptRepoS3Token); HRN_CFG_LOAD(cfgCmdArchivePush, argList);