diff --git a/README.md b/README.md index 02ac1d8a..5cdb488b 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,18 @@ acr purge \ --ago 30d \ --concurrency 4 ``` + +#### Repository page size flag +To control the number of repositories fetched in a single page, the `--repository-page-size` flag should be set. A default value of 100 will be used if `--repository-page-size` is not specified. +This is useful when the number of artifacts in the registry is very large and listing too many repositories at once can timeout. +```sh +acr purge \ + --registry \ + --filter : \ + --ago 30d \ + --repository-page-size 10 +``` + ### Integration with ACR Tasks To run a locally built version of the ACR-CLI using ACR Tasks follow these steps: diff --git a/cmd/acr/annotate.go b/cmd/acr/annotate.go index 622b8ac1..9c4e47bf 100644 --- a/cmd/acr/annotate.go +++ b/cmd/acr/annotate.go @@ -82,7 +82,7 @@ func newAnnotateCmd(rootParams *rootParameters) *cobra.Command { } // A map is used to collect the regex tags for every repository. - tagFilters, err := common.CollectTagFilters(ctx, annotateParams.filters, acrClient.AutorestClient, annotateParams.filterTimeout) + tagFilters, err := common.CollectTagFilters(ctx, annotateParams.filters, acrClient.AutorestClient, annotateParams.filterTimeout, defaultRepoPageSize) if err != nil { return err } diff --git a/cmd/acr/purge.go b/cmd/acr/purge.go index ae38ec7d..94b369a1 100644 --- a/cmd/acr/purge.go +++ b/cmd/acr/purge.go @@ -46,14 +46,19 @@ const ( - Delete all tags older than 1 day in the example.azurecr.io registry inside the hello-world repository, with 4 purge tasks running concurrently acr purge -r example --filter "hello-world:.*" --ago 1d --concurrency 4 + + - Delete all tags that are older than 7 days in the example.azurecr.io registry inside all repositories, with a page size of 50 repositories + acr purge -r example --filter ".*:.*" --ago 7d --repository-page-size 50 ` maxPoolSize = 32 // The max number of parallel delete requests recommended by ACR server headerLink = "Link" ) var ( - defaultPoolSize = runtime.GOMAXPROCS(0) - concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize) + defaultPoolSize = runtime.GOMAXPROCS(0) + defaultRepoPageSize = int32(100) + repoPageSizeDescription = "Number of repositories queried at once" + concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize) ) // Default settings for regexp2 @@ -71,6 +76,7 @@ type purgeParameters struct { untagged bool dryRun bool concurrency int + repoPageSize int32 } // newPurgeCmd defines the purge command. @@ -95,7 +101,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command { return err } // A map is used to collect the regex tags for every repository. - tagFilters, err := common.CollectTagFilters(ctx, purgeParams.filters, acrClient.AutorestClient, purgeParams.filterTimeout) + tagFilters, err := common.CollectTagFilters(ctx, purgeParams.filters, acrClient.AutorestClient, purgeParams.filterTimeout, purgeParams.repoPageSize) if err != nil { return err } @@ -162,6 +168,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command { cmd.Flags().StringArrayVarP(&purgeParams.configs, "config", "c", nil, "Authentication config paths (e.g. C://Users/docker/config.json)") cmd.Flags().Int64Var(&purgeParams.filterTimeout, "filter-timeout-seconds", defaultRegexpMatchTimeoutSeconds, "This limits the evaluation of the regex filter, and will return a timeout error if this duration is exceeded during a single evaluation. If written incorrectly a regexp filter with backtracking can result in an infinite loop.") cmd.Flags().IntVar(&purgeParams.concurrency, "concurrency", defaultPoolSize, concurrencyDescription) + cmd.Flags().Int32Var(&purgeParams.repoPageSize, "repository-page-size", defaultRepoPageSize, repoPageSizeDescription) cmd.Flags().BoolP("help", "h", false, "Print usage") cmd.MarkFlagRequired("filter") cmd.MarkFlagRequired("ago") diff --git a/cmd/acr/purge_test.go b/cmd/acr/purge_test.go index 55e55439..a9f8c6e1 100644 --- a/cmd/acr/purge_test.go +++ b/cmd/acr/purge_test.go @@ -713,7 +713,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local[.].+"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local[.].+"}, mockClient, 60, defaultRepoPageSize) assert.Equal(4, len(filters), "Number of found should be 4") assert.Equal(".*-?local[.].+", filters[testRepo], "Filter for test repo should be .*-?local[.].+") assert.Equal(".*-?local[.].+", filters["bar"], "Filter for bar repo should be .*-?local[.].+") @@ -726,7 +726,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local\\..+"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local\\..+"}, mockClient, 60, defaultRepoPageSize) assert.Equal(4, len(filters), "Number of found should be 4") assert.Equal(".*-?local\\..+", filters[testRepo], "Filter for test repo should be .*-?local\\..+") assert.Equal(".*-?local\\..+", filters["bar"], "Filter for bar repo should be .*-?local\\..+") @@ -739,7 +739,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found should be one") assert.Equal(".*", filters[testRepo], "Filter for test repo should be .*") assert.Equal(nil, err, "Error should be nil") @@ -751,7 +751,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{".*:.*"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{".*:.*"}, mockClient, 60, defaultRepoPageSize) assert.Equal(4, len(filters), "Number of found should be 4") assert.Equal(".*", filters[testRepo], "Filter for test repo should be .*") assert.Equal(".*", filters["bar"], "Filter for bar repo should be .*") @@ -764,7 +764,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"ba:.*"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"ba:.*"}, mockClient, 60, defaultRepoPageSize) assert.Equal(0, len(filters), "Number of found repos should be zero") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -775,7 +775,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:.*"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:.*"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found repos should be one") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -786,7 +786,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:(?:.*)"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:(?:.*)"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found repos should be one") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -797,7 +797,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):(?:.*)"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):(?:.*)"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found repos should be one") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -808,7 +808,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*)?:(?:.*)"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*)?:(?:.*)"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found repos should be one") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -819,7 +819,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):.(?:.*)"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):.(?:.*)"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found repos should be one") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -830,7 +830,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{"foo/b[[:alpha:]]r(?:.*):.(?:.*)"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{"foo/b[[:alpha:]]r(?:.*):.(?:.*)"}, mockClient, 60, defaultRepoPageSize) assert.Equal(1, len(filters), "Number of found repos should be one") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -840,7 +840,7 @@ func TestCollectTagFilters(t *testing.T) { assert := assert.New(t) mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(NoRepositoriesResult, nil).Once() - filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60) + filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60, defaultRepoPageSize) assert.Equal(0, len(filters), "Number of found repos should be zero") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -851,7 +851,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - _, err := common.CollectTagFilters(testCtx, []string{":.*"}, mockClient, 60) + _, err := common.CollectTagFilters(testCtx, []string{":.*"}, mockClient, 60, defaultRepoPageSize) assert.NotEqual(nil, err, "Error should not be nil") mockClient.AssertExpectations(t) }) @@ -861,7 +861,7 @@ func TestCollectTagFilters(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - _, err := common.CollectTagFilters(testCtx, []string{testRepo + ".*:"}, mockClient, 60) + _, err := common.CollectTagFilters(testCtx, []string{testRepo + ".*:"}, mockClient, 60, defaultRepoPageSize) assert.NotEqual(nil, err, "Error should not be nil") mockClient.AssertExpectations(t) }) @@ -873,7 +873,7 @@ func TestGetAllRepositoryNames(t *testing.T) { mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient) + allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize) assert.Equal(4, len(allRepoNames), "Number of all repo names should be 4") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -885,7 +885,7 @@ func TestGetAllRepositoryNames(t *testing.T) { mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(MoreRepositoriesResult, nil).Once() mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once() - allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient) + allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize) assert.Equal(7, len(allRepoNames), "Number of all repo names should be 7") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) @@ -895,7 +895,7 @@ func TestGetAllRepositoryNames(t *testing.T) { assert := assert.New(t) mockClient := &mocks.BaseClientAPI{} mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(NoRepositoriesResult, nil).Once() - allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient) + allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize) assert.Equal(0, len(allRepoNames), "Number of all repo names should be 7") assert.Equal(nil, err, "Error should be nil") mockClient.AssertExpectations(t) diff --git a/cmd/common/image_functions.go b/cmd/common/image_functions.go index 6192989b..e3179caf 100644 --- a/cmd/common/image_functions.go +++ b/cmd/common/image_functions.go @@ -29,12 +29,11 @@ const ( mediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json" ) -func GetAllRepositoryNames(ctx context.Context, client acrapi.BaseClientAPI) ([]string, error) { +func GetAllRepositoryNames(ctx context.Context, client acrapi.BaseClientAPI, pageSize int32) ([]string, error) { allRepoNames := make([]string, 0) lastName := "" - var batchSize int32 = 100 for { - repos, err := client.GetRepositories(ctx, lastName, &batchSize) + repos, err := client.GetRepositories(ctx, lastName, &pageSize) if err != nil { return nil, err } @@ -97,8 +96,8 @@ func GetRepositoryAndTagRegex(filter string) (string, string, error) { } // CollectTagFilters collects all matching repos and collects the associated tag filters -func CollectTagFilters(ctx context.Context, rawFilters []string, client acrapi.BaseClientAPI, regexMatchTimeout int64) (map[string]string, error) { - allRepoNames, err := GetAllRepositoryNames(ctx, client) +func CollectTagFilters(ctx context.Context, rawFilters []string, client acrapi.BaseClientAPI, regexMatchTimeout int64, repoPageSize int32) (map[string]string, error) { + allRepoNames, err := GetAllRepositoryNames(ctx, client, repoPageSize) if err != nil { return nil, err }