fix(cleanup): ignore harbor "unsupported 404 status code" errors

These errors may occur when harbor configured to use S3 backend, which may respond with the following error:

```
GET https://domain/harbor/s3/object/name/prefix/docker/registry/v2/blobs/sha256/2d/3d8c68cd9df32f1beb4392298a123eac5
8aba1433a15b3258b2f3728bad4b7d1/data?X-Amz-Algorithm=REDACTED&X-Amz-Credential=REDACTED&X-Amz-Date=REDACTED&X-Amz-Expire
s=REDACTED&X-Amz-Signature=REDACTED&X-Amz-SignedHeaders=REDACTED: unsupported status code 404; body: <?xml version="1.0"
encoding="UTF-8"?>
<Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Resource>/harbor/s3/object/name/pr
efix/docker/registry/v2/blobs/sha256/3d/3d8c68cd9df32f1beb4392298a123eac58aba1433a15b3258b2f3728bad4b7d1/data</Resource>
<RequestId>c5bb943c-1e85-5930-b455-c3e8edbbaccd</RequestId></Error>
```

pkg/docker_registry/api.go

@@ -81,7 +81,7 @@ func (api *api) IsRepoImageExists(ctx context.Context, reference string) (bool,
 
 func (api *api) TryGetRepoImage(ctx context.Context, reference string) (*image.Info, error) {
 	if imgInfo, err := api.GetRepoImage(ctx, reference); err != nil {
-		if IsBlobUnknownError(err) || IsManifestUnknownError(err) || IsNameUnknownError(err) {
+		if IsBlobUnknownError(err) || IsManifestUnknownError(err) || IsNameUnknownError(err) || IsHarbor404Error(err) {
 			// TODO: 1. auto reject images with manifest-unknown or blob-unknown errors
 			// TODO: 2. why TryGetRepoImage for rejected image records gives manifest-unknown errors?
 			// TODO: 3. make sure werf never ever creates rejected image records for name-unknown errors.

pkg/docker_registry/default.go

@@ -55,3 +55,15 @@ func IsBlobUnknownError(err error) bool {
 func IsNameUnknownError(err error) bool {
 	return (err != nil) && strings.Contains(err.Error(), "NAME_UNKNOWN")
 }
+
+func IsHarbor404Error(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Example error:
+	// GET https://domain/harbor/s3/object/name/prefix/docker/registry/v2/blobs/sha256/2d/3d8c68cd9df32f1beb4392298a123eac58aba1433a15b3258b2f3728bad4b7d1/data?X-Amz-Algorithm=REDACTED&X-Amz-Credential=REDACTED&X-Amz-Date=REDACTED&X-Amz-Expires=REDACTED&X-Amz-Signature=REDACTED&X-Amz-SignedHeaders=REDACTED: unsupported status code 404; body: <?xml version="1.0" encoding="UTF-8"?>
+	// <Error><Code>NoSuchKey</Code><Message>The specified key does not exist.</Message><Resource>/harbor/s3/object/name/prefix/docker/registry/v2/blobs/sha256/3d/3d8c68cd9df32f1beb4392298a123eac58aba1433a15b3258b2f3728bad4b7d1/data</Resource><RequestId>c5bb943c-1e85-5930-b455-c3e8edbbaccd</RequestId></Error>
+
+	return strings.Contains(err.Error(), "unsupported status code 404")
+}

pkg/docker_registry/generic_api.go

@@ -49,7 +49,7 @@ func (api *genericApi) GetRepoImageConfigFile(ctx context.Context, reference str
 	for _, mirrorReference := range mirrorReferenceList {
 		config, err := api.getRepoImageConfigFile(ctx, mirrorReference)
 		if err != nil {
-			if IsBlobUnknownError(err) || IsManifestUnknownError(err) || IsNameUnknownError(err) {
+			if IsBlobUnknownError(err) || IsManifestUnknownError(err) || IsNameUnknownError(err) || IsHarbor404Error(err) {
 				continue
 			}
 

pkg/storage/repo_stages_storage.go

@@ -273,7 +273,7 @@ func (storage *RepoStagesStorage) GetStageDescription(ctx context.Context, proje
 		return nil, nil
 	}
 
-	if docker_registry.IsBlobUnknownError(err) {
+	if docker_registry.IsBlobUnknownError(err) || docker_registry.IsHarbor404Error(err) {
 		return nil, ErrBrokenImage
 	}