Skip to content

Make the number of repository fetched at once configurable to handle large registries #353

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
estebanreyl merged 3 commits into from
Oct 11, 2024
Merged

Make the number of repository fetched at once configurable to handle large registries #353

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6ce115c
Make the number of repository fetched at once configurable to handle …
JRBANCEL Sep 17, 2024
357c770
Rename batch to page
JRBANCEL Sep 23, 2024
f233e4e
Update the documentation/examples
JRBANCEL Sep 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,18 @@ acr purge \
--ago 30d \
--concurrency 4
```

#### Repository page size flag
To control the number of repositories fetched in a single page, the `--repository-page-size` flag should be set. A default value of 100 will be used if `--repository-page-size` is not specified.
This is useful when the number of artifacts in the registry is very large and listing too many repositories at once can timeout.
```sh
acr purge \
--registry <Registry Name> \
--filter <Repository Filter/Name>:<Regex Filter> \
--ago 30d \
--repository-page-size 10
```

### Integration with ACR Tasks

To run a locally built version of the ACR-CLI using ACR Tasks follow these steps:
Expand Down
2 changes: 1 addition & 1 deletion cmd/acr/annotate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func newAnnotateCmd(rootParams *rootParameters) *cobra.Command {
}

// A map is used to collect the regex tags for every repository.
tagFilters, err := common.CollectTagFilters(ctx, annotateParams.filters, acrClient.AutorestClient, annotateParams.filterTimeout)
tagFilters, err := common.CollectTagFilters(ctx, annotateParams.filters, acrClient.AutorestClient, annotateParams.filterTimeout, defaultRepoPageSize)
if err != nil {
return err
}
Expand Down
13 changes: 10 additions & 3 deletions cmd/acr/purge.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,19 @@ const (

- Delete all tags older than 1 day in the example.azurecr.io registry inside the hello-world repository, with 4 purge tasks running concurrently
acr purge -r example --filter "hello-world:.*" --ago 1d --concurrency 4

- Delete all tags that are older than 7 days in the example.azurecr.io registry inside all repositories, with a page size of 50 repositories
acr purge -r example --filter ".*:.*" --ago 7d --repository-page-size 50
`
maxPoolSize = 32 // The max number of parallel delete requests recommended by ACR server
headerLink = "Link"
)

var (
defaultPoolSize = runtime.GOMAXPROCS(0)
concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize)
defaultPoolSize = runtime.GOMAXPROCS(0)
defaultRepoPageSize = int32(100)
repoPageSizeDescription = "Number of repositories queried at once"
concurrencyDescription = fmt.Sprintf("Number of concurrent purge tasks. Range: [1 - %d]", maxPoolSize)
)

// Default settings for regexp2
Expand All @@ -71,6 +76,7 @@ type purgeParameters struct {
untagged bool
dryRun bool
concurrency int
repoPageSize int32
}

// newPurgeCmd defines the purge command.
Expand All @@ -95,7 +101,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
return err
}
// A map is used to collect the regex tags for every repository.
tagFilters, err := common.CollectTagFilters(ctx, purgeParams.filters, acrClient.AutorestClient, purgeParams.filterTimeout)
tagFilters, err := common.CollectTagFilters(ctx, purgeParams.filters, acrClient.AutorestClient, purgeParams.filterTimeout, purgeParams.repoPageSize)
if err != nil {
return err
}
Expand Down Expand Up @@ -162,6 +168,7 @@ func newPurgeCmd(rootParams *rootParameters) *cobra.Command {
cmd.Flags().StringArrayVarP(&purgeParams.configs, "config", "c", nil, "Authentication config paths (e.g. C://Users/docker/config.json)")
cmd.Flags().Int64Var(&purgeParams.filterTimeout, "filter-timeout-seconds", defaultRegexpMatchTimeoutSeconds, "This limits the evaluation of the regex filter, and will return a timeout error if this duration is exceeded during a single evaluation. If written incorrectly a regexp filter with backtracking can result in an infinite loop.")
cmd.Flags().IntVar(&purgeParams.concurrency, "concurrency", defaultPoolSize, concurrencyDescription)
cmd.Flags().Int32Var(&purgeParams.repoPageSize, "repository-page-size", defaultRepoPageSize, repoPageSizeDescription)
cmd.Flags().BoolP("help", "h", false, "Print usage")
cmd.MarkFlagRequired("filter")
cmd.MarkFlagRequired("ago")
Expand Down
34 changes: 17 additions & 17 deletions cmd/acr/purge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local[.].+"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local[.].+"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(4, len(filters), "Number of found should be 4")
assert.Equal(".*-?local[.].+", filters[testRepo], "Filter for test repo should be .*-?local[.].+")
assert.Equal(".*-?local[.].+", filters["bar"], "Filter for bar repo should be .*-?local[.].+")
Expand All @@ -726,7 +726,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local\\..+"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{".+:.*-?local\\..+"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(4, len(filters), "Number of found should be 4")
assert.Equal(".*-?local\\..+", filters[testRepo], "Filter for test repo should be .*-?local\\..+")
assert.Equal(".*-?local\\..+", filters["bar"], "Filter for bar repo should be .*-?local\\..+")
Expand All @@ -739,7 +739,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found should be one")
assert.Equal(".*", filters[testRepo], "Filter for test repo should be .*")
assert.Equal(nil, err, "Error should be nil")
Expand All @@ -751,7 +751,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{".*:.*"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{".*:.*"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(4, len(filters), "Number of found should be 4")
assert.Equal(".*", filters[testRepo], "Filter for test repo should be .*")
assert.Equal(".*", filters["bar"], "Filter for bar repo should be .*")
Expand All @@ -764,7 +764,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"ba:.*"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"ba:.*"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(0, len(filters), "Number of found repos should be zero")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -775,7 +775,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:.*"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:.*"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found repos should be one")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -786,7 +786,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:(?:.*)"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar:(?:.*)"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found repos should be one")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -797,7 +797,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):(?:.*)"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):(?:.*)"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found repos should be one")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -808,7 +808,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*)?:(?:.*)"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*)?:(?:.*)"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found repos should be one")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -819,7 +819,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):.(?:.*)"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"foo/bar(?:.*):.(?:.*)"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found repos should be one")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -830,7 +830,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{"foo/b[[:alpha:]]r(?:.*):.(?:.*)"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{"foo/b[[:alpha:]]r(?:.*):.(?:.*)"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(1, len(filters), "Number of found repos should be one")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -840,7 +840,7 @@ func TestCollectTagFilters(t *testing.T) {
assert := assert.New(t)
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(NoRepositoriesResult, nil).Once()
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60)
filters, err := common.CollectTagFilters(testCtx, []string{testRepo + ":.*"}, mockClient, 60, defaultRepoPageSize)
assert.Equal(0, len(filters), "Number of found repos should be zero")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -851,7 +851,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
_, err := common.CollectTagFilters(testCtx, []string{":.*"}, mockClient, 60)
_, err := common.CollectTagFilters(testCtx, []string{":.*"}, mockClient, 60, defaultRepoPageSize)
assert.NotEqual(nil, err, "Error should not be nil")
mockClient.AssertExpectations(t)
})
Expand All @@ -861,7 +861,7 @@ func TestCollectTagFilters(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
_, err := common.CollectTagFilters(testCtx, []string{testRepo + ".*:"}, mockClient, 60)
_, err := common.CollectTagFilters(testCtx, []string{testRepo + ".*:"}, mockClient, 60, defaultRepoPageSize)
assert.NotEqual(nil, err, "Error should not be nil")
mockClient.AssertExpectations(t)
})
Expand All @@ -873,7 +873,7 @@ func TestGetAllRepositoryNames(t *testing.T) {
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient)
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize)
assert.Equal(4, len(allRepoNames), "Number of all repo names should be 4")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -885,7 +885,7 @@ func TestGetAllRepositoryNames(t *testing.T) {
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(ManyRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(MoreRepositoriesResult, nil).Once()
mockClient.On("GetRepositories", mock.Anything, mock.Anything, mock.Anything).Return(NoRepositoriesResult, nil).Once()
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient)
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize)
assert.Equal(7, len(allRepoNames), "Number of all repo names should be 7")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand All @@ -895,7 +895,7 @@ func TestGetAllRepositoryNames(t *testing.T) {
assert := assert.New(t)
mockClient := &mocks.BaseClientAPI{}
mockClient.On("GetRepositories", mock.Anything, "", mock.Anything).Return(NoRepositoriesResult, nil).Once()
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient)
allRepoNames, err := common.GetAllRepositoryNames(testCtx, mockClient, defaultRepoPageSize)
assert.Equal(0, len(allRepoNames), "Number of all repo names should be 7")
assert.Equal(nil, err, "Error should be nil")
mockClient.AssertExpectations(t)
Expand Down
9 changes: 4 additions & 5 deletions cmd/common/image_functions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,11 @@ const (
mediaTypeArtifactManifest = "application/vnd.oci.artifact.manifest.v1+json"
)

func GetAllRepositoryNames(ctx context.Context, client acrapi.BaseClientAPI) ([]string, error) {
func GetAllRepositoryNames(ctx context.Context, client acrapi.BaseClientAPI, pageSize int32) ([]string, error) {
allRepoNames := make([]string, 0)
lastName := ""
var batchSize int32 = 100
for {
repos, err := client.GetRepositories(ctx, lastName, &batchSize)
repos, err := client.GetRepositories(ctx, lastName, &pageSize)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -97,8 +96,8 @@ func GetRepositoryAndTagRegex(filter string) (string, string, error) {
}

// CollectTagFilters collects all matching repos and collects the associated tag filters
func CollectTagFilters(ctx context.Context, rawFilters []string, client acrapi.BaseClientAPI, regexMatchTimeout int64) (map[string]string, error) {
allRepoNames, err := GetAllRepositoryNames(ctx, client)
func CollectTagFilters(ctx context.Context, rawFilters []string, client acrapi.BaseClientAPI, regexMatchTimeout int64, repoPageSize int32) (map[string]string, error) {
allRepoNames, err := GetAllRepositoryNames(ctx, client, repoPageSize)
if err != nil {
return nil, err
}
Expand Down