Merge pull request #8 from Ahuge/ah/chore/update

Ah/chore/update

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  # Dependencies listed in go.mod
+  - package-ecosystem: "gomod"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+    # Create security fix PRs only
+    open-pull-requests-limit: 0
+
+
+  # Dependencies listed in .github/workflows/*.yml
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    # Create security fix PRs only
+    open-pull-requests-limit: 0

.github/workflows/ci.yml

@@ -1,13 +1,18 @@
 name: ci
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
 jobs:
   build:
     strategy:
       matrix:
         go-version:
+          - 1.20.x
           - 1.19.x
-          - 1.18.x
-          - 1.17.x
         os:
           - macos
           - ubuntu
@@ -27,29 +32,42 @@ jobs:
     strategy:
       matrix:
         go-version:
+          - 1.20.x
           - 1.19.x
-          - 1.18.x
-          - 1.17.x
         os:
           - macos
           - ubuntu
           - windows
 
     name: test (${{ matrix.os }}/go-${{ matrix.go-version }})
     runs-on: ${{ matrix.os }}-latest
+    services:
+      minio:
+        image: ${{ (matrix.os == 'ubuntu') && 'bitnami/minio:2023.7.18' || ''}}
+        ports:
+          - 45677:9000
+        options: >-
+          --health-cmd "curl -I http://localhost:9000/minio/health/live -s"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
 
     - run: make test
-
+      env:
+        S5CMD_TEST_ENDPOINT_URL: ${{ (matrix.os == 'ubuntu') && 'http://localhost:45677' || '' }}
   qa:
     strategy:
       matrix:
         go-version:
-          - 1.19.x
+          - 1.20.x
         os:
           - ubuntu
 
@@ -61,8 +79,10 @@ jobs:
         go-version: ${{ matrix.go-version }}
 
     - run: make check-fmt
+    - run: go install github.com/golang/mock/mockgen@v1.6.0
+    - run: make check-codegen
     - run: make vet
-    - run: go install honnef.co/go/tools/cmd/staticcheck@v0.3.0
+    - run: go install honnef.co/go/tools/cmd/staticcheck@v0.4.3
     - run: make staticcheck
     - run: go install mvdan.cc/unparam@latest
     - run: make unparam

.github/workflows/docker.yml

@@ -28,7 +28,7 @@ jobs:
           context: .
           file: ./Dockerfile
           push: true
-          platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7,linux/386
+          platforms: linux/amd64,linux/arm64,linux/ppc64le,linux/arm/v6,linux/arm/v7,linux/386
           tags: |
             peakcom/s5cmd:latest
             peakcom/s5cmd:${{ github.event.release.tag_name }}

.github/workflows/goreleaser.yml

@@ -4,7 +4,25 @@ on:
     tags:
       - 'v*'
 jobs:
+  gcs_test:
+    name: gcs-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.20.x
+      - run: make test_without_race
+        env:
+          S5CMD_ACCESS_KEY_ID: ${{ secrets.S5CMD_GCS_ACCESS_KEY_ID }}
+          S5CMD_SECRET_ACCESS_KEY: ${{ secrets.S5CMD_GCS_SECRET_ACCESS_KEY }}
+          S5CMD_ENDPOINT_URL: https://storage.googleapis.com
+          S5CMD_IS_VIRTUAL_HOST: 1
+          S5CMD_REGION: ${{ secrets.S5CMD_GCS_REGION }}
+          S5CMD_I_KNOW_WHAT_IM_DOING: 1
+
   goreleaser:
+    needs: [ gcs_test ]
     runs-on: ubuntu-latest
     steps:
       -
@@ -16,12 +34,12 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19
+          go-version: '1.20'
       -
         name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:
-          version: latest
+          version: '1.18.2'
           args: release --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }}

.goreleaser.yml

@@ -4,7 +4,7 @@ before:
 builds:
   -
     binary: s5cmd
-    ldflags: -s -w -X github.com/peak/s5cmd/version.Version={{.Tag}} -X github.com/peak/s5cmd/version.GitCommit={{ .ShortCommit }}
+    ldflags: -s -w -X github.com/peak/s5cmd/v2/version.Version={{.Tag}} -X github.com/peak/s5cmd/v2/version.GitCommit={{ .ShortCommit }}
     env:
       - CGO_ENABLED=0
     goos:
@@ -16,6 +16,7 @@ builds:
       - amd64
       - arm
       - arm64
+      - ppc64le
     ignore:
       - goos: darwin
         goarch: 386

CHANGELOG.md

@@ -1,6 +1,42 @@
 # Changelog
+## v2.2.2 - 13 Sep 2023 
 
-## not released yet
+#### Bugfixes
+- Fixed `cp` and `pipe` to not omit some of the metadata flags. ([#657](https://github.com/peak/s5cmd/issues/657))
+## v2.2.1 - 23 Aug 2023
+
+#### Bugfixes
+- Fixed incorrect `s5cmd version` output ([#650](https://github.com/peak/s5cmd/pull/650))
+## v2.2.0 - 21 Aug 2023
+
+#### Features
+- Added `pipe` command. ([#182](https://github.com/peak/s5cmd/issues/182))
+- Added `presign` command. ([#634](https://github.com/peak/s5cmd/pull/634)) [@zemul](https://github.com/zemul)
+- Added file types to `select` queries with more range of options to set during the query. ([#494](https://github.com/peak/s5cmd/issues/494))
+- Added `--content-disposition` flag to `cp` command. ([#569](https://github.com/peak/s5cmd/issues/569))
+- Added `--show-fullpath` flag to `ls` command. ([#596](https://github.com/peak/s5cmd/issues/596))
+- Added `--show-progress` flag to `cp` command. ([#51](https://github.com/peak/s5cmd/issues/51))
+- Added `--metadata` flag to `cp` and `pipe` commands to set arbitrary metadata for the objects. ([#537](https://github.com/peak/s5cmd/issues/537))
+- Added `--include` flag to `cp`, `rm`, and `sync` commands. ([#516](https://github.com/peak/s5cmd/issues/516))
+- Added `--content-disposition` flag to `cp` command. ([#569](https://github.com/peak/s5cmd/issues/569))
+
+
+#### Improvements
+- Implemented concurrent multipart download support for `cat` command. ([#245](https://github.com/peak/s5cmd/issues/245))
+- Upgraded minimum required Go version to 1.19. ([#583](https://github.com/peak/s5cmd/pull/583))
+- `ListObjectsV2` S3 API is enabled for Google Cloud Storage. ([#617](https://github.com/peak/s5cmd/pull/617))
+- Added installation instructions for FreeBSD. ([#573](https://github.com/peak/s5cmd/pull/573)) [@ehaupt](https://github.com/ehaupt)
+- Added `ppc64le` support. ([#552](https://github.com/peak/s5cmd/pull/552)) [@mgiessing](https://github.com/mgiessing)
+
+#### Bugfixes
+- Fixed a bug that causes `sync` command with whitespaced flag value to fail. ([#541](https://github.com/peak/s5cmd/issues/541)) [ataberkgrl](https://github.com/ataberkgrl)
+- Fixed a bug introduced with `external sort` support in `sync` command which prevents `sync` to an empty destination with `--delete` option. ([#576](https://github.com/peak/s5cmd/issues/576))
+- Fixed a bug in `sync` command, which previously caused the command to continue running even if an error was received from the destination bucket. ([#564](https://github.com/peak/s5cmd/issues/564))
+- Fixed a bug that causes local files to be lost if downloads fail. ([#479](https://github.com/peak/s5cmd/issues/479))
+- Fixed a bug where `cp` command could not upload a non-regular file to remote destination. ([#618](https://github.com/peak/s5cmd/pull/618))
+- Fixed a crash where a file or a remote object is removed or renamed after it is listed to be operated on. ([#620](https://github.com/peak/s5cmd/pull/620))
+
+## v2.1.0 - 19 Jun 2023
 
 #### Breaking changes
 - Adjacent slashes in key are no longer removed when uploading to remote. Before `s5cmd cp file.txt s3://bucket/a//b///c/` would copy to `s3://bucket/a/b/c/file.txt` but now to `s3://bucket/a//b///c/file.txt`.([#459](https://github.com/peak/s5cmd/pull/459))
@@ -10,13 +46,21 @@
 - Added `--content-type` and `--content-encoding` flags to `cp` command. ([#264](https://github.com/peak/s5cmd/issues/264))
 - Added `--profile` flag to allow users to specify a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). ([#353](https://github.com/peak/s5cmd/issues/353))
 - Added `--credentials-file` flag to allow users to specify path for the AWS credentials file instead of using the [default location](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where).
+- Added `--all-versions` flag to `ls`, `rm`, `du` and `select` subcommands to apply operation on(/over) all versions of the objects. ([#475](https://github.com/peak/s5cmd/pull/475))
+- Added `--version-id` flag to `cat`, `cp`/`mv`, `rm`, `du`  and `select` subcommands to apply operation on(/over) a specific versions of the object. ([#475](https://github.com/peak/s5cmd/pull/475))
+- Added `bucket-version` command to configure bucket versioning. Bucket name alone returns the bucket versioning status of the bucket. Bucket versioning can be configured with `set` flag. ([#475](https://github.com/peak/s5cmd/pull/475))
+- Added `--raw` flag to `cat` and `select` subcommands. It disables the wildcard operations. ([#475](https://github.com/peak/s5cmd/pull/475))
 - Added `bench.py` script under new `benchmark` folder to compare performances of two different builds of s5cmd. ([#471](https://github.com/peak/s5cmd/pull/471))
 
 #### Improvements
-- Disable AWS SDK logger if log level is not `trace`. ([##460](https://github.com/peak/s5cmd/pull/460))
+- Disable AWS SDK logger if log level is not `trace`. ([#460](https://github.com/peak/s5cmd/pull/460))
 - Allow adjacent slashes to be used as keys when uploading to remote. ([#459](https://github.com/peak/s5cmd/pull/459))
 - Debian packages are provided on [releases page](https://github.com/peak/s5cmd/releases) ([#380](https://github.com/peak/s5cmd/issues/380))
 - Upgraded minimum required Go version to 1.17.
+- The sync command uses `external sort` instead of `internal` sort. This change
+ reduces RAM usage from ~10 GB to ~1.5 GB for `sync` operation of a directory containing
+ 1,000,000 files at a cost of speed (20% slower for 1,000,000 objects). For smaller
+ directories (~50,000 files) there is no significant change in speed.  ([#483](https://github.com/peak/s5cmd/pull/483))
 - Improve auto-completion support of s5cmd for `zsh` and `bash`, start supporting `pwsh` and stop the support for `fish`. Now s5cmd can complete bucket names, s3 keys in a bucket and the local files. However, `install-completion` flag no longer _installs_ the completion script to `*rc` files instead it merely gives instructions to install autocompletion and provides the autocompletion script ([#500](https://github.com/peak/s5cmd/pull/500)).
 
 #### Bugfixes

Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.19-alpine as build
+FROM golang:1.20-alpine as build
 COPY . /s5cmd/
 RUN apk add --no-cache git make && \
     cd /s5cmd/ && \
     CGO_ENABLED=0 make build
 
-FROM alpine:3.15
+FROM alpine:3.18
 COPY --from=build /s5cmd/s5cmd .
 WORKDIR /aws
 ENTRYPOINT ["/s5cmd"]

Makefile

@@ -5,7 +5,7 @@ all: clean build test check
 
 VERSION := `git describe --abbrev=0 --tags || echo "0.0.0"`
 BUILD := `git rev-parse --short HEAD`
-LDFLAGS=-ldflags "-X=github.com/peak/s5cmd/version.Version=$(VERSION) -X=github.com/peak/s5cmd/version.GitCommit=$(BUILD)"
+LDFLAGS=-ldflags "-X=github.com/peak/s5cmd/v2/version.Version=$(VERSION) -X=github.com/peak/s5cmd/v2/version.GitCommit=$(BUILD)"
 
 .PHONY: build
 build:
@@ -28,11 +28,11 @@ test_without_race:
 	@S5CMD_BUILD_BINARY_WITHOUT_RACE_FLAG=1 go test -mod=vendor -count=1 ./...
 
 .PHONY: check
-check: vet staticcheck unparam check-fmt
+check: vet staticcheck unparam check-fmt check-codegen
 
 .PHONY: staticcheck
 staticcheck:
-	@staticcheck -checks 'all,-U1000,-ST1000,-ST1003' ./...
+	@staticcheck -checks 'all,-ST1000' ./...
 
 .PHONY: unparam
 unparam:
@@ -49,9 +49,13 @@ check-fmt:
 		exit 1;\
 	fi
 
-.PHONY: mock
-mock:
-	@mockery -inpkg -dir=storage -name=Storage -case=underscore
+.PHONY: check-codegen
+check-codegen: gogenerate ## Check generated code is up-to-date
+	@git diff --exit-code --
+
+.PHONY: gogenerate
+gogenerate:
+	@go generate -mod vendor ./...
 
 .PHONY: clean
 clean: