Skip to content

Commit ff4eff2

Browse files
feat(workflows): cleanup cloudsmith and docker repos
1 parent 8f951fc commit ff4eff2

7 files changed

Lines changed: 1193 additions & 0 deletions

File tree

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
const CLOUDSMITH_API = 'https://api.cloudsmith.io/v1';
2+
const DEFAULT_GRACE_MS = 24 * 60 * 60 * 1000;
3+
4+
/**
5+
* Return the package version used by native packages for a GitHub release tag.
6+
*
7+
* @param {string} tag Release tag.
8+
* @returns {string} Native package version prefix.
9+
*/
10+
export function releaseVersion(tag) {
11+
return String(tag ?? '').trim().replace(/^v(?=\d)/i, '');
12+
}
13+
14+
/**
15+
* Determine whether a native package version belongs to a GitHub release.
16+
*
17+
* DEB and RPM package versions append their packaging release after a hyphen.
18+
* Exact matches cover formats which do not add a packaging release.
19+
*
20+
* @param {string} packageVersion Cloudsmith package version.
21+
* @param {Iterable<string>} releaseTags Published GitHub release tags.
22+
* @returns {boolean} Whether the package belongs to a published release.
23+
*/
24+
export function packageMatchesRelease(packageVersion, releaseTags) {
25+
const candidate = String(packageVersion ?? '').trim();
26+
if (!candidate) {
27+
return false;
28+
}
29+
30+
for (const tag of releaseTags) {
31+
const version = releaseVersion(tag);
32+
if (version && (candidate === version || candidate.startsWith(`${version}-`))) {
33+
return true;
34+
}
35+
}
36+
return false;
37+
}
38+
39+
/**
40+
* Check all Cloudsmith tag groups for a tag.
41+
*
42+
* @param {object} packageData Cloudsmith package.
43+
* @param {string} tag Package tag.
44+
* @returns {boolean} Whether the tag exists.
45+
*/
46+
export function hasCloudsmithTag(packageData, tag) {
47+
return Object.values(packageData.tags ?? {}).some(
48+
(values) => Array.isArray(values) && values.includes(tag),
49+
);
50+
}
51+
52+
/**
53+
* Return published stable and prerelease tags, excluding drafts.
54+
*
55+
* @param {Array<object>} releases GitHub releases.
56+
* @returns {Set<string>} Published release tags.
57+
*/
58+
export function publishedReleaseTags(releases) {
59+
return new Set(
60+
releases
61+
.filter((release) => !release.draft && release.tag_name)
62+
.map((release) => release.tag_name),
63+
);
64+
}
65+
66+
function isRecent(timestamp, now, graceMs) {
67+
const time = Date.parse(timestamp ?? '');
68+
return Number.isFinite(time) && now - time < graceMs;
69+
}
70+
71+
/**
72+
* Build a fail-closed Cloudsmith cleanup plan.
73+
*
74+
* Package names map case-insensitively to GitHub repository names. Unknown
75+
* package names and packages which Cloudsmith cannot currently delete are kept.
76+
*
77+
* @param {object} options Planning options.
78+
* @param {Array<object>} options.packages Cloudsmith packages.
79+
* @param {Array<object>} options.repositories GitHub repositories.
80+
* @param {Map<string, Set<string>>} options.releaseTagsByRepository Tags keyed by lower-case repo name.
81+
* @param {number} [options.now] Current time in milliseconds.
82+
* @param {number} [options.graceMs] New-package grace period.
83+
* @returns {{keep: Array<object>, remove: Array<object>}} Cleanup plan.
84+
*/
85+
export function planCloudsmithCleanup({
86+
packages,
87+
repositories,
88+
releaseTagsByRepository,
89+
now = Date.now(),
90+
graceMs = DEFAULT_GRACE_MS,
91+
}) {
92+
const repositoryNames = new Map(
93+
repositories.map((repository) => [repository.name.toLowerCase(), repository.name]),
94+
);
95+
const plan = {keep: [], remove: []};
96+
97+
for (const packageData of packages) {
98+
const packageName = String(packageData.name ?? '');
99+
const repositoryName = repositoryNames.get(packageName.toLowerCase());
100+
let reason;
101+
102+
if (!repositoryName) {
103+
reason = 'unknown GitHub repository';
104+
} else if (!packageData.identifier_perm && !packageData.slug_perm) {
105+
reason = 'missing Cloudsmith identifier';
106+
} else if (packageData.is_deleteable === false) {
107+
reason = 'not deleteable';
108+
} else if (hasCloudsmithTag(packageData, 'latest')) {
109+
reason = 'latest package for target';
110+
} else if (isRecent(packageData.uploaded_at, now, graceMs)) {
111+
reason = 'within publish grace period';
112+
} else {
113+
const tags = releaseTagsByRepository.get(repositoryName.toLowerCase()) ?? new Set();
114+
if (packageMatchesRelease(packageData.version_orig ?? packageData.version, tags)) {
115+
reason = 'published GitHub release';
116+
}
117+
}
118+
119+
const item = {package: packageData, repository: repositoryName, reason};
120+
if (reason) {
121+
plan.keep.push(item);
122+
} else {
123+
plan.remove.push(item);
124+
}
125+
}
126+
127+
return plan;
128+
}
129+
130+
async function responseError(response) {
131+
const body = await response.text();
132+
return new Error(`${response.status} ${response.statusText}: ${body.slice(0, 500)}`);
133+
}
134+
135+
async function listCloudsmithPackages({owner, repository, token}) {
136+
const packages = [];
137+
const pageSize = 100;
138+
139+
for (let page = 1; ; page += 1) {
140+
const url = new URL(`${CLOUDSMITH_API}/packages/${owner}/${repository}/`);
141+
url.searchParams.set('page', page);
142+
url.searchParams.set('page_size', pageSize);
143+
const response = await fetch(url, {
144+
headers: {
145+
Authorization: `Bearer ${token}`,
146+
'User-Agent': 'LizardByte-artifact-cleanup',
147+
},
148+
});
149+
if (!response.ok) {
150+
throw await responseError(response);
151+
}
152+
153+
const pagePackages = await response.json();
154+
packages.push(...pagePackages);
155+
const lastPage = Number(response.headers.get('x-pagination-pagetotal'));
156+
if ((Number.isFinite(lastPage) && page >= lastPage) || pagePackages.length < pageSize) {
157+
break;
158+
}
159+
}
160+
161+
return packages;
162+
}
163+
164+
async function deleteCloudsmithPackage({owner, repository, token, identifier}) {
165+
const response = await fetch(
166+
`${CLOUDSMITH_API}/packages/${owner}/${repository}/${encodeURIComponent(identifier)}/`,
167+
{
168+
method: 'DELETE',
169+
headers: {
170+
Authorization: `Bearer ${token}`,
171+
'User-Agent': 'LizardByte-artifact-cleanup',
172+
},
173+
},
174+
);
175+
if (!response.ok && response.status !== 404) {
176+
throw await responseError(response);
177+
}
178+
}
179+
180+
async function loadRepositories(github, organization) {
181+
return github.paginate(github.rest.repos.listForOrg, {
182+
org: organization,
183+
per_page: 100,
184+
type: 'all',
185+
});
186+
}
187+
188+
async function loadReleaseTags(github, organization, repositoryNames) {
189+
const tagsByRepository = new Map();
190+
for (const repository of repositoryNames) {
191+
const releases = await github.paginate(github.rest.repos.listReleases, {
192+
owner: organization,
193+
repo: repository,
194+
per_page: 100,
195+
});
196+
tagsByRepository.set(repository.toLowerCase(), publishedReleaseTags(releases));
197+
}
198+
return tagsByRepository;
199+
}
200+
201+
function packageLabel(packageData) {
202+
const architecture = (packageData.architectures ?? []).map(({name}) => name).join(',');
203+
const target = [
204+
packageData.format,
205+
packageData.distro?.slug,
206+
packageData.distro_version?.slug,
207+
architecture,
208+
].filter(Boolean).join('/');
209+
return `${packageData.name} ${packageData.version} (${target})`;
210+
}
211+
212+
/**
213+
* Clean a Cloudsmith repository against the organization's published releases.
214+
*
215+
* @param {object} options Runtime dependencies and settings.
216+
* @returns {Promise<object>} Cleanup counts.
217+
*/
218+
export async function cleanupCloudsmith({
219+
github,
220+
core,
221+
token,
222+
dryRun,
223+
organization = 'LizardByte',
224+
owner = 'lizardbyte',
225+
repository = 'beta',
226+
}) {
227+
const [packages, repositories] = await Promise.all([
228+
listCloudsmithPackages({owner, repository, token}),
229+
loadRepositories(github, organization),
230+
]);
231+
const knownNames = new Set(repositories.map(({name}) => name.toLowerCase()));
232+
const matchedRepositories = new Set(
233+
packages
234+
.map(({name}) => repositories.find(
235+
(repositoryData) => repositoryData.name.toLowerCase() === String(name).toLowerCase(),
236+
)?.name)
237+
.filter(Boolean),
238+
);
239+
const releaseTagsByRepository = await loadReleaseTags(
240+
github,
241+
organization,
242+
[...matchedRepositories],
243+
);
244+
const plan = planCloudsmithCleanup({packages, repositories, releaseTagsByRepository});
245+
246+
for (const {package: packageData, reason} of plan.keep) {
247+
if (reason === 'unknown GitHub repository') {
248+
core.warning(`Keeping ${packageLabel(packageData)}: ${reason}.`);
249+
}
250+
}
251+
252+
for (const {package: packageData} of plan.remove) {
253+
const identifier = packageData.identifier_perm ?? packageData.slug_perm;
254+
if (dryRun) {
255+
core.info(`[dry-run] Delete ${packageLabel(packageData)} (${identifier}).`);
256+
} else {
257+
core.info(`Deleting ${packageLabel(packageData)} (${identifier}).`);
258+
await deleteCloudsmithPackage({owner, repository, token, identifier});
259+
}
260+
}
261+
262+
const unknownNames = new Set(
263+
packages
264+
.map(({name}) => String(name).toLowerCase())
265+
.filter((name) => !knownNames.has(name)),
266+
);
267+
await core.summary
268+
.addHeading(`Cloudsmith ${owner}/${repository} cleanup`, 2)
269+
.addTable([
270+
[{data: 'Mode', header: true}, {data: dryRun ? 'Dry run' : 'Delete'}],
271+
[{data: 'Packages scanned', header: true}, {data: String(packages.length)}],
272+
[{data: 'Packages retained', header: true}, {data: String(plan.keep.length)}],
273+
[{data: 'Packages selected', header: true}, {data: String(plan.remove.length)}],
274+
[{data: 'Unknown package names', header: true}, {data: String(unknownNames.size)}],
275+
])
276+
.write();
277+
278+
return {
279+
scanned: packages.length,
280+
retained: plan.keep.length,
281+
selected: plan.remove.length,
282+
};
283+
}

0 commit comments

Comments
 (0)