Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add pgbackrest wal-fetch and wal-show commands (#1202)
* Write pgbackrest wal-fetch and wal-show commands * Test pgbakcrest wal-fetch functionality * Fix lint * Download file without extension as last resort * Fix pgbench command * Fix build * Fix pgbackrest test * Update PostgreSQL.md * Update PostgreSQL.md * Add minor adjustments to DownloadAndDecompressStorageFile Co-authored-by: usernamedt <usernamedt@yandex-team.com>
- Loading branch information
1 parent
dfa01e4
commit 1eb88a5
Showing
12 changed files
with
327 additions
and
44 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package pg | ||
|
|
||
| import ( | ||
| "github.com/spf13/cobra" | ||
| "github.com/wal-g/tracelog" | ||
| "github.com/wal-g/wal-g/internal/databases/postgres/pgbackrest" | ||
| ) | ||
|
|
||
| var pgbackrestWalFetchCmd = &cobra.Command{ | ||
| Use: "wal-fetch wal_name destination_filename", | ||
| Short: WalFetchShortDescription, | ||
| Args: cobra.ExactArgs(2), | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| folder, stanza := configurePgbackrestSettings() | ||
| err := pgbackrest.HandleWalFetch(folder, stanza, args[0], args[1]) | ||
| tracelog.ErrorLogger.FatalOnError(err) | ||
| }, | ||
| } | ||
|
|
||
| func init() { | ||
| pgbackrestCmd.AddCommand(pgbackrestWalFetchCmd) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| package pg | ||
|
|
||
| import ( | ||
| "os" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "github.com/wal-g/tracelog" | ||
| "github.com/wal-g/wal-g/internal/databases/postgres" | ||
| "github.com/wal-g/wal-g/internal/databases/postgres/pgbackrest" | ||
| ) | ||
|
|
||
| var pgbackrestWalgShowCmd = &cobra.Command{ | ||
| Use: "wal-show", | ||
| Short: WalShowUsage, | ||
| Long: WalShowLongDescription, | ||
| Args: cobra.NoArgs, | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| folder, stanza := configurePgbackrestSettings() | ||
| outputType := postgres.TableOutput | ||
| if detailedJSONOutput { | ||
| outputType = postgres.JSONOutput | ||
| } | ||
| outputWriter := postgres.NewWalShowOutputWriter(outputType, os.Stdout, false) | ||
| err := pgbackrest.HandleWalShow(folder, stanza, outputWriter) | ||
| tracelog.ErrorLogger.FatalOnError(err) | ||
| }, | ||
| } | ||
|
|
||
| func init() { | ||
| pgbackrestCmd.AddCommand(pgbackrestWalgShowCmd) | ||
| pgbackrestWalgShowCmd.Flags().BoolVar(&detailedJSONOutput, detailedOutputFlag, false, detailedOutputDescription) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
41 changes: 41 additions & 0 deletions
41
internal/databases/postgres/pgbackrest/wal_fetch_handler.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package pgbackrest | ||
|
|
||
| import ( | ||
| "errors" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/wal-g/wal-g/internal" | ||
| "github.com/wal-g/wal-g/pkg/storages/storage" | ||
| ) | ||
|
|
||
| func HandleWalFetch(folder storage.Folder, stanza string, walFileName string, location string) error { | ||
| archiveName, err := GetArchiveName(folder, stanza) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| archiveFolder := folder.GetSubFolder(WalArchivePath).GetSubFolder(stanza).GetSubFolder(*archiveName) | ||
| if strings.HasSuffix(walFileName, ".history") { | ||
| return internal.DownloadFileTo(archiveFolder, walFileName, location) | ||
| } | ||
|
|
||
| subdirectoryName := walFileName[0:16] | ||
| walFolder := archiveFolder.GetSubFolder(subdirectoryName) | ||
| if strings.HasSuffix(walFileName, ".backup") { | ||
| return internal.DownloadFileTo(walFolder, walFileName, location) | ||
| } | ||
| fileList, _, err := walFolder.ListFolder() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| for _, file := range fileList { | ||
| fileName := file.GetName() | ||
| if strings.HasPrefix(fileName, walFileName) { | ||
| return internal.DownloadFileTo(walFolder, strings.TrimSuffix(fileName, filepath.Ext(fileName)), location) | ||
| } | ||
| } | ||
|
|
||
| return errors.New("File " + walFileName + " not found in storage") | ||
| } |
101 changes: 101 additions & 0 deletions
101
internal/databases/postgres/pgbackrest/wal_show_handler.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| package pgbackrest | ||
|
|
||
| import ( | ||
| "path" | ||
| "sort" | ||
| "strings" | ||
|
|
||
| "github.com/wal-g/tracelog" | ||
| "github.com/wal-g/wal-g/internal/databases/postgres" | ||
| "github.com/wal-g/wal-g/pkg/storages/storage" | ||
| ) | ||
|
|
||
| func HandleWalShow(rootFolder storage.Folder, stanza string, outputWriter postgres.WalShowOutputWriter) error { | ||
| archiveName, err := GetArchiveName(rootFolder, stanza) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| archiveFolder := rootFolder.GetSubFolder(WalArchivePath).GetSubFolder(stanza).GetSubFolder(*archiveName) | ||
| walFiles, err := getWalFiles(archiveFolder) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| walSegments, err := getWalSegments(walFiles) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| walSequencesByTimelines := getWalSequencesByTimelines(walSegments) | ||
|
|
||
| var timelineInfos []*postgres.TimelineInfo | ||
| for _, segmentsSequence := range walSequencesByTimelines { | ||
| historyRecords, err := postgres.GetTimeLineHistoryRecords(segmentsSequence.TimelineID, archiveFolder) | ||
| if err != nil { | ||
| if _, ok := err.(postgres.HistoryFileNotFoundError); !ok { | ||
| tracelog.ErrorLogger.Fatalf("Error while loading .history file %v\n", err) | ||
| } | ||
| } | ||
|
|
||
| info, err := postgres.NewTimelineInfo(segmentsSequence, historyRecords) | ||
| tracelog.ErrorLogger.FatalfOnError("Error while creating TimeLineInfo %v\n", err) | ||
| timelineInfos = append(timelineInfos, info) | ||
| } | ||
|
|
||
| sort.Slice(timelineInfos, func(i, j int) bool { | ||
| return timelineInfos[i].ID < timelineInfos[j].ID | ||
| }) | ||
|
|
||
| return outputWriter.Write(timelineInfos) | ||
| } | ||
|
|
||
| func getWalSequencesByTimelines(segments []postgres.WalSegmentDescription) map[uint32]*postgres.WalSegmentsSequence { | ||
| segmentsByTimelines := make(map[uint32]*postgres.WalSegmentsSequence) | ||
| for _, segment := range segments { | ||
| if timelineInfo, ok := segmentsByTimelines[segment.Timeline]; ok { | ||
| timelineInfo.AddWalSegmentNo(segment.Number) | ||
| continue | ||
| } | ||
| segmentsByTimelines[segment.Timeline] = postgres.NewSegmentsSequence(segment.Timeline, segment.Number) | ||
| } | ||
| return segmentsByTimelines | ||
| } | ||
|
|
||
| func getWalSegments(filenames []string) ([]postgres.WalSegmentDescription, error) { | ||
| var segments []postgres.WalSegmentDescription | ||
| for _, filename := range filenames { | ||
| extension := path.Ext(filename) | ||
| if extension == ".backup" || extension == ".history" { | ||
| continue | ||
| } | ||
|
|
||
| segmentName := strings.Split(path.Base(filename), "-")[0] | ||
| segment, err := postgres.NewWalSegmentDescription(segmentName) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| segments = append(segments, segment) | ||
| } | ||
| return segments, nil | ||
| } | ||
|
|
||
| func getWalFiles(archiveFolder storage.Folder) ([]string, error) { | ||
| var walFiles []string | ||
| _, walDirectories, err := archiveFolder.ListFolder() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| for _, walDirectory := range walDirectories { | ||
| files, _, err := walDirectory.ListFolder() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| for _, file := range files { | ||
| walFiles = append(walFiles, path.Join(walDirectory.GetPath(), file.GetName())) | ||
| } | ||
| } | ||
| return walFiles, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.